logstash-core 5.0.0.alpha6.snapshot1-java → 5.0.0.alpha6.snapshot2-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of logstash-core might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/lib/jars.rb +7 -0
- data/lib/logstash-core/version.rb +1 -1
- data/lib/logstash/agent.rb +2 -2
- data/lib/logstash/api/rack_app.rb +14 -15
- data/lib/logstash/config/config_ast.rb +2 -2
- data/lib/logstash/config/file.rb +1 -2
- data/lib/logstash/config/mixin.rb +9 -11
- data/lib/logstash/environment.rb +2 -1
- data/lib/logstash/filters/base.rb +1 -0
- data/lib/logstash/inputs/base.rb +1 -0
- data/lib/logstash/java_integration.rb +1 -0
- data/lib/logstash/logging.rb +1 -89
- data/lib/logstash/logging/json.rb +1 -1
- data/lib/logstash/logging/logger.rb +72 -0
- data/lib/logstash/output_delegator.rb +8 -16
- data/lib/logstash/output_delegator_strategies/legacy.rb +2 -1
- data/lib/logstash/output_delegator_strategies/shared.rb +1 -0
- data/lib/logstash/output_delegator_strategies/single.rb +1 -0
- data/lib/logstash/outputs/base.rb +5 -0
- data/lib/logstash/patches/clamp.rb +21 -0
- data/lib/logstash/pipeline.rb +24 -24
- data/lib/logstash/pipeline_reporter.rb +1 -1
- data/lib/logstash/plugin.rb +3 -10
- data/lib/logstash/plugins/registry.rb +3 -1
- data/lib/logstash/runner.rb +47 -74
- data/lib/logstash/shutdown_watcher.rb +1 -8
- data/lib/logstash/util/decorators.rb +3 -4
- data/lib/logstash/util/java_version.rb +0 -5
- data/lib/logstash/util/loggable.rb +7 -17
- data/lib/logstash/util/safe_uri.rb +2 -0
- data/lib/logstash/util/wrapped_synchronous_queue.rb +5 -5
- data/lib/logstash/version.rb +1 -1
- data/locales/en.yml +19 -8
- data/logstash-core.gemspec +4 -3
- data/spec/api/lib/rack_app_spec.rb +6 -4
- data/spec/logstash/config/loader_spec.rb +3 -1
- data/spec/logstash/config/mixin_spec.rb +28 -9
- data/spec/logstash/output_delegator_spec.rb +15 -2
- data/spec/logstash/outputs/base_spec.rb +0 -1
- data/spec/logstash/pipeline_reporter_spec.rb +1 -0
- data/spec/logstash/pipeline_spec.rb +14 -2
- data/spec/logstash/plugin_spec.rb +7 -6
- data/spec/logstash/runner_spec.rb +62 -44
- data/spec/logstash/shutdown_watcher_spec.rb +0 -4
- metadata +21 -18
@@ -5,22 +5,24 @@ require "logstash/output_delegator_strategies/single"
|
|
5
5
|
require "logstash/output_delegator_strategies/legacy"
|
6
6
|
|
7
7
|
module LogStash class OutputDelegator
|
8
|
-
attr_reader :metric, :metric_events, :strategy, :namespaced_metric, :metric_events
|
8
|
+
attr_reader :metric, :metric_events, :strategy, :namespaced_metric, :metric_events, :id
|
9
9
|
|
10
10
|
def initialize(logger, output_class, metric, strategy_registry, plugin_args)
|
11
11
|
@logger = logger
|
12
12
|
@output_class = output_class
|
13
13
|
@metric = metric
|
14
|
-
@
|
15
|
-
|
14
|
+
@id = plugin_args["id"]
|
15
|
+
|
16
16
|
raise ArgumentError, "No strategy registry specified" unless strategy_registry
|
17
17
|
raise ArgumentError, "No ID specified! Got args #{plugin_args}" unless id
|
18
18
|
|
19
|
-
|
20
|
-
|
19
|
+
@strategy = strategy_registry.
|
20
|
+
class_for(self.concurrency).
|
21
|
+
new(@logger, @output_class, @metric, plugin_args)
|
22
|
+
|
21
23
|
@namespaced_metric = metric.namespace(id.to_sym)
|
24
|
+
@namespaced_metric.gauge(:name, config_name)
|
22
25
|
@metric_events = @namespaced_metric.namespace(:events)
|
23
|
-
@namespaced_metric.gauge(:name, id)
|
24
26
|
end
|
25
27
|
|
26
28
|
def config_name
|
@@ -31,16 +33,6 @@ module LogStash class OutputDelegator
|
|
31
33
|
@output_class.concurrency
|
32
34
|
end
|
33
35
|
|
34
|
-
def build_strategy!
|
35
|
-
@strategy = strategy_registry.
|
36
|
-
class_for(self.concurrency).
|
37
|
-
new(@logger, @output_class, @metric, @plugin_args)
|
38
|
-
end
|
39
|
-
|
40
|
-
def id
|
41
|
-
@plugin_args["id"]
|
42
|
-
end
|
43
|
-
|
44
36
|
def register
|
45
37
|
@strategy.register
|
46
38
|
end
|
@@ -4,7 +4,8 @@ module LogStash module OutputDelegatorStrategies class Legacy
|
|
4
4
|
|
5
5
|
def initialize(logger, klass, metric, plugin_args)
|
6
6
|
@worker_count = (plugin_args["workers"] || 1).to_i
|
7
|
-
@workers = @worker_count.times.map {
|
7
|
+
@workers = @worker_count.times.map { klass.new(plugin_args) }
|
8
|
+
@workers.each {|w| w.metric = metric }
|
8
9
|
@worker_queue = SizedQueue.new(@worker_count)
|
9
10
|
@workers.each {|w| @worker_queue << w}
|
10
11
|
end
|
@@ -8,6 +8,7 @@ require "logstash/util/wrapped_synchronous_queue"
|
|
8
8
|
require "concurrent/atomic/atomic_fixnum"
|
9
9
|
|
10
10
|
class LogStash::Outputs::Base < LogStash::Plugin
|
11
|
+
include LogStash::Util::Loggable
|
11
12
|
include LogStash::Config::Mixin
|
12
13
|
|
13
14
|
config_name "output"
|
@@ -92,6 +93,10 @@ class LogStash::Outputs::Base < LogStash::Plugin
|
|
92
93
|
end
|
93
94
|
end
|
94
95
|
|
96
|
+
def workers_not_supported(message=nil)
|
97
|
+
raise "This plugin (#{self.class.name}) is using the obsolete '#workers_not_supported' method. If you installed this plugin specifically on this Logstash version, it is not compatible. If you are a plugin author, please see https://www.elastic.co/guide/en/logstash/current/_how_to_write_a_logstash_output_plugin.html for more info"
|
98
|
+
end
|
99
|
+
|
95
100
|
def codec
|
96
101
|
params["codec"]
|
97
102
|
end
|
@@ -16,9 +16,20 @@ module Clamp
|
|
16
16
|
|
17
17
|
module Option
|
18
18
|
|
19
|
+
module Declaration
|
20
|
+
def deprecated_option(switches, type, description, opts = {})
|
21
|
+
Option::Definition.new(switches, type, description, opts).tap do |option|
|
22
|
+
declared_options << option
|
23
|
+
block ||= option.default_conversion_block
|
24
|
+
define_deprecated_accessors_for(option, opts, &block)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
19
29
|
module StrictDeclaration
|
20
30
|
|
21
31
|
include Clamp::Attribute::Declaration
|
32
|
+
include LogStash::Util::Loggable
|
22
33
|
|
23
34
|
# Instead of letting Clamp set up accessors for the options
|
24
35
|
# weŕe going to tightly controlling them through
|
@@ -37,6 +48,16 @@ module Clamp
|
|
37
48
|
end
|
38
49
|
end
|
39
50
|
|
51
|
+
def define_deprecated_accessors_for(option, opts, &block)
|
52
|
+
define_deprecated_writer_for(option, opts, &block)
|
53
|
+
end
|
54
|
+
|
55
|
+
def define_deprecated_writer_for(option, opts, &block)
|
56
|
+
define_method(option.write_method) do |value|
|
57
|
+
self.class.logger.warn "DEPRECATION WARNING: The flag #{option.switches} has been deprecated, please use \"--#{opts[:new_flag]}=#{opts[:new_value]}\" instead."
|
58
|
+
LogStash::SETTINGS.set(opts[:new_flag], opts[:new_value])
|
59
|
+
end
|
60
|
+
end
|
40
61
|
end
|
41
62
|
|
42
63
|
class Definition
|
data/lib/logstash/pipeline.rb
CHANGED
@@ -20,6 +20,8 @@ require "logstash/output_delegator"
|
|
20
20
|
require "logstash/filter_delegator"
|
21
21
|
|
22
22
|
module LogStash; class Pipeline
|
23
|
+
include LogStash::Util::Loggable
|
24
|
+
|
23
25
|
attr_reader :inputs,
|
24
26
|
:filters,
|
25
27
|
:outputs,
|
@@ -28,7 +30,6 @@ module LogStash; class Pipeline
|
|
28
30
|
:events_filtered,
|
29
31
|
:reporter,
|
30
32
|
:pipeline_id,
|
31
|
-
:logger,
|
32
33
|
:started_at,
|
33
34
|
:thread,
|
34
35
|
:config_str,
|
@@ -44,17 +45,16 @@ module LogStash; class Pipeline
|
|
44
45
|
"LogStash::Inputs::Stdin"
|
45
46
|
]
|
46
47
|
|
47
|
-
def initialize(config_str, settings =
|
48
|
+
def initialize(config_str, settings = SETTINGS, namespaced_metric = nil)
|
49
|
+
@logger = self.logger
|
48
50
|
@config_str = config_str
|
49
51
|
@config_hash = Digest::SHA1.hexdigest(@config_str)
|
50
52
|
# Every time #plugin is invoked this is incremented to give each plugin
|
51
53
|
# a unique id when auto-generating plugin ids
|
52
|
-
@plugin_counter ||= 0
|
53
|
-
|
54
|
-
@logger = Cabin::Channel.get(LogStash)
|
54
|
+
@plugin_counter ||= 0
|
55
55
|
@settings = settings
|
56
56
|
@pipeline_id = @settings.get_value("pipeline.id") || self.object_id
|
57
|
-
@reporter =
|
57
|
+
@reporter = PipelineReporter.new(@logger, self)
|
58
58
|
|
59
59
|
# A list of plugins indexed by id
|
60
60
|
@plugins_by_id = {}
|
@@ -66,12 +66,12 @@ module LogStash; class Pipeline
|
|
66
66
|
|
67
67
|
# This needs to be configured before we evaluate the code to make
|
68
68
|
# sure the metric instance is correctly send to the plugins to make the namespace scoping work
|
69
|
-
@metric = namespaced_metric.nil? ?
|
69
|
+
@metric = namespaced_metric.nil? ? Instrument::NullMetric.new : namespaced_metric
|
70
70
|
|
71
71
|
grammar = LogStashConfigParser.new
|
72
72
|
@config = grammar.parse(config_str)
|
73
73
|
if @config.nil?
|
74
|
-
raise
|
74
|
+
raise ConfigurationError, grammar.failure_reason
|
75
75
|
end
|
76
76
|
# This will compile the config to ruby and evaluate the resulting code.
|
77
77
|
# The code will initialize all the plugins and define the
|
@@ -82,8 +82,8 @@ module LogStash; class Pipeline
|
|
82
82
|
# The config code is hard to represent as a log message...
|
83
83
|
# So just print it.
|
84
84
|
|
85
|
-
if @settings.get_value("config.debug") && logger.debug?
|
86
|
-
logger.debug("Compiled pipeline code", :code => code)
|
85
|
+
if @settings.get_value("config.debug") && @logger.debug?
|
86
|
+
@logger.debug("Compiled pipeline code", :code => code)
|
87
87
|
end
|
88
88
|
|
89
89
|
begin
|
@@ -92,7 +92,7 @@ module LogStash; class Pipeline
|
|
92
92
|
raise
|
93
93
|
end
|
94
94
|
|
95
|
-
queue =
|
95
|
+
queue = Util::WrappedSynchronousQueue.new
|
96
96
|
@input_queue_client = queue.write_client
|
97
97
|
@filter_queue_client = queue.read_client
|
98
98
|
# Note that @inflight_batches as a central mechanism for tracking inflight
|
@@ -148,11 +148,11 @@ module LogStash; class Pipeline
|
|
148
148
|
@started_at = Time.now
|
149
149
|
|
150
150
|
@thread = Thread.current
|
151
|
-
|
151
|
+
Util.set_thread_name("[#{pipeline_id}]-pipeline-manager")
|
152
152
|
|
153
153
|
start_workers
|
154
154
|
|
155
|
-
@logger.
|
155
|
+
@logger.info("Pipeline #{@pipeline_id} started")
|
156
156
|
|
157
157
|
# Block until all inputs have stopped
|
158
158
|
# Generally this happens if SIGINT is sent and `shutdown` is called from an external thread
|
@@ -167,7 +167,7 @@ module LogStash; class Pipeline
|
|
167
167
|
shutdown_flusher
|
168
168
|
shutdown_workers
|
169
169
|
|
170
|
-
@logger.
|
170
|
+
@logger.info("Pipeline #{@pipeline_id} has been shutdown")
|
171
171
|
|
172
172
|
# exit code
|
173
173
|
return 0
|
@@ -221,7 +221,7 @@ module LogStash; class Pipeline
|
|
221
221
|
|
222
222
|
pipeline_workers.times do |t|
|
223
223
|
@worker_threads << Thread.new do
|
224
|
-
|
224
|
+
Util.set_thread_name("[#{pipeline_id}]>worker#{t}")
|
225
225
|
worker_loop(batch_size, batch_delay)
|
226
226
|
end
|
227
227
|
end
|
@@ -256,7 +256,7 @@ module LogStash; class Pipeline
|
|
256
256
|
|
257
257
|
def filter_batch(batch)
|
258
258
|
batch.each do |event|
|
259
|
-
if event.is_a?(
|
259
|
+
if event.is_a?(Event)
|
260
260
|
filtered = filter_func(event)
|
261
261
|
filtered.each do |e|
|
262
262
|
#these are both original and generated events
|
@@ -331,7 +331,7 @@ module LogStash; class Pipeline
|
|
331
331
|
end
|
332
332
|
|
333
333
|
def inputworker(plugin)
|
334
|
-
|
334
|
+
Util::set_thread_name("[#{pipeline_id}]<#{plugin.class.config_name}")
|
335
335
|
begin
|
336
336
|
plugin.run(@input_queue_client)
|
337
337
|
rescue => e
|
@@ -387,7 +387,7 @@ module LogStash; class Pipeline
|
|
387
387
|
# Each worker thread will receive this exactly once!
|
388
388
|
@worker_threads.each do |t|
|
389
389
|
@logger.debug("Pushing shutdown", :thread => t.inspect)
|
390
|
-
@input_queue_client.push(
|
390
|
+
@input_queue_client.push(SHUTDOWN)
|
391
391
|
end
|
392
392
|
|
393
393
|
@worker_threads.each do |t|
|
@@ -411,20 +411,20 @@ module LogStash; class Pipeline
|
|
411
411
|
args["id"]
|
412
412
|
end
|
413
413
|
|
414
|
-
raise
|
414
|
+
raise ConfigurationError, "Two plugins have the id '#{id}', please fix this conflict" if @plugins_by_id[id]
|
415
415
|
|
416
416
|
pipeline_scoped_metric = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :plugins])
|
417
417
|
|
418
|
-
klass =
|
418
|
+
klass = Plugin.lookup(plugin_type, name)
|
419
419
|
|
420
420
|
# Scope plugins of type 'input' to 'inputs'
|
421
421
|
type_scoped_metric = pipeline_scoped_metric.namespace("#{plugin_type}s".to_sym)
|
422
422
|
plugin = if plugin_type == "output"
|
423
423
|
OutputDelegator.new(@logger, klass, type_scoped_metric,
|
424
|
-
|
424
|
+
OutputDelegatorStrategyRegistry.instance,
|
425
425
|
args)
|
426
426
|
elsif plugin_type == "filter"
|
427
|
-
|
427
|
+
FilterDelegator.new(@logger, klass, type_scoped_metric, args)
|
428
428
|
else # input
|
429
429
|
input_plugin = klass.new(args)
|
430
430
|
input_plugin.metric = type_scoped_metric.namespace(id)
|
@@ -472,7 +472,7 @@ module LogStash; class Pipeline
|
|
472
472
|
def flush
|
473
473
|
if @flushing.compare_and_set(false, true)
|
474
474
|
@logger.debug? && @logger.debug("Pushing flush onto pipeline")
|
475
|
-
@input_queue_client.push(
|
475
|
+
@input_queue_client.push(FLUSH)
|
476
476
|
end
|
477
477
|
end
|
478
478
|
|
@@ -506,7 +506,7 @@ module LogStash; class Pipeline
|
|
506
506
|
def plugin_threads_info
|
507
507
|
input_threads = @input_threads.select {|t| t.alive? }
|
508
508
|
worker_threads = @worker_threads.select {|t| t.alive? }
|
509
|
-
(input_threads + worker_threads).map {|t|
|
509
|
+
(input_threads + worker_threads).map {|t| Util.thread_info(t) }
|
510
510
|
end
|
511
511
|
|
512
512
|
def stalling_threads_info
|
@@ -101,7 +101,7 @@ module LogStash; class PipelineReporter
|
|
101
101
|
pipeline.outputs.map do |output_delegator|
|
102
102
|
{
|
103
103
|
:type => output_delegator.config_name,
|
104
|
-
:
|
104
|
+
:id => output_delegator.id,
|
105
105
|
:concurrency => output_delegator.concurrency,
|
106
106
|
}
|
107
107
|
end
|
data/lib/logstash/plugin.rb
CHANGED
@@ -3,14 +3,13 @@ require "logstash/namespace"
|
|
3
3
|
require "logstash/logging"
|
4
4
|
require "logstash/config/mixin"
|
5
5
|
require "logstash/instrument/null_metric"
|
6
|
-
require "cabin"
|
7
6
|
require "concurrent"
|
8
7
|
require "securerandom"
|
9
8
|
require "logstash/plugins/registry"
|
10
9
|
|
11
10
|
class LogStash::Plugin
|
11
|
+
include LogStash::Util::Loggable
|
12
12
|
attr_accessor :params
|
13
|
-
attr_accessor :logger
|
14
13
|
|
15
14
|
NL = "\n"
|
16
15
|
|
@@ -45,12 +44,12 @@ class LogStash::Plugin
|
|
45
44
|
self.class.name == other.class.name && @params == other.params
|
46
45
|
end
|
47
46
|
|
48
|
-
def initialize(params=nil)
|
47
|
+
def initialize(params=nil)
|
48
|
+
@logger = self.logger
|
49
49
|
@params = LogStash::Util.deep_clone(params)
|
50
50
|
# The id should always be defined normally, but in tests that might not be the case
|
51
51
|
# In the future we may make this more strict in the Plugin API
|
52
52
|
@params["id"] ||= "#{self.class.config_name}_#{SecureRandom.uuid}"
|
53
|
-
@logger = Cabin::Channel.get(LogStash)
|
54
53
|
end
|
55
54
|
|
56
55
|
# Return a uniq ID for this plugin configuration, by default
|
@@ -161,10 +160,4 @@ class LogStash::Plugin
|
|
161
160
|
def self.is_a_plugin?(klass, name)
|
162
161
|
klass.ancestors.include?(LogStash::Plugin) && klass.respond_to?(:config_name) && klass.config_name == name
|
163
162
|
end
|
164
|
-
|
165
|
-
# @return [Cabin::Channel] logger channel for class methods
|
166
|
-
def self.logger
|
167
|
-
@logger ||= Cabin::Channel.get(LogStash)
|
168
|
-
end
|
169
|
-
|
170
163
|
end # class LogStash::Plugin
|
@@ -1,9 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'singleton'
|
3
3
|
require "rubygems/package"
|
4
|
+
require "logstash/util/loggable"
|
4
5
|
|
5
6
|
module LogStash
|
6
7
|
class Registry
|
8
|
+
include LogStash::Util::Loggable
|
7
9
|
|
8
10
|
##
|
9
11
|
# Placeholder class for registered plugins
|
@@ -45,7 +47,7 @@ module LogStash
|
|
45
47
|
|
46
48
|
def initialize
|
47
49
|
@registry = {}
|
48
|
-
@logger =
|
50
|
+
@logger = self.logger
|
49
51
|
end
|
50
52
|
|
51
53
|
def lookup(type, plugin_name, &block)
|
data/lib/logstash/runner.rb
CHANGED
@@ -4,7 +4,6 @@ Encoding.default_external = Encoding::UTF_8
|
|
4
4
|
$DEBUGLIST = (ENV["DEBUG"] || "").split(",")
|
5
5
|
|
6
6
|
require "clamp"
|
7
|
-
require "cabin"
|
8
7
|
require "net/http"
|
9
8
|
require "logstash/environment"
|
10
9
|
|
@@ -19,9 +18,10 @@ require "logstash/settings"
|
|
19
18
|
require "logstash/version"
|
20
19
|
|
21
20
|
class LogStash::Runner < Clamp::StrictCommand
|
21
|
+
include LogStash::Util::Loggable
|
22
22
|
# The `path.settings` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
|
23
23
|
# because the `Environment::LOGSTASH_HOME` doesn't exist in the context of the `logstash-core` gem.
|
24
|
-
#
|
24
|
+
#
|
25
25
|
# See issue https://github.com/elastic/logstash/issues/5361
|
26
26
|
LogStash::SETTINGS.register(LogStash::Setting::String.new("path.settings", ::File.join(LogStash::Environment::LOGSTASH_HOME, "config")))
|
27
27
|
|
@@ -82,7 +82,8 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
82
82
|
:attribute_name => "path.log"
|
83
83
|
|
84
84
|
option "--log.level", "LEVEL", I18n.t("logstash.runner.flag.log_level"),
|
85
|
-
:default => LogStash::SETTINGS.get_default("log.level")
|
85
|
+
:default => LogStash::SETTINGS.get_default("log.level"),
|
86
|
+
:attribute_name => "log.level"
|
86
87
|
|
87
88
|
option "--config.debug", :flag,
|
88
89
|
I18n.t("logstash.runner.flag.config_debug"),
|
@@ -132,10 +133,22 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
132
133
|
:attribute_name => "path.settings",
|
133
134
|
:default => LogStash::SETTINGS.get_default("path.settings")
|
134
135
|
|
136
|
+
### DEPRECATED FLAGS ###
|
137
|
+
deprecated_option ["--verbose"], :flag,
|
138
|
+
I18n.t("logstash.runner.flag.verbose"),
|
139
|
+
:new_flag => "log.level", :new_value => "info"
|
140
|
+
|
141
|
+
deprecated_option ["--debug"], :flag,
|
142
|
+
I18n.t("logstash.runner.flag.debug"),
|
143
|
+
:new_flag => "log.level", :new_value => "debug"
|
144
|
+
|
145
|
+
deprecated_option ["--quiet"], :flag,
|
146
|
+
I18n.t("logstash.runner.flag.quiet"),
|
147
|
+
:new_flag => "log.level", :new_value => "error"
|
148
|
+
|
135
149
|
attr_reader :agent
|
136
150
|
|
137
151
|
def initialize(*args)
|
138
|
-
@logger = Cabin::Channel.get(LogStash)
|
139
152
|
@settings = LogStash::SETTINGS
|
140
153
|
super(*args)
|
141
154
|
end
|
@@ -150,13 +163,20 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
150
163
|
rescue => e
|
151
164
|
# abort unless we're just looking for the help
|
152
165
|
if (["--help", "-h"] & args).empty?
|
153
|
-
|
154
|
-
|
155
|
-
@logger.fatal("Failed to load settings file from \"path.settings\". Aborting...", "path.settings" => LogStash::SETTINGS.get("path.settings"), "exception" => e.class, "message" => e.message)
|
166
|
+
$stderr.puts "INFO: Logstash has a new settings file which defines start up time settings. This file is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually please specify the location to this settings file by passing in \"--path.settings=/path/..\" in the command line options"
|
167
|
+
$stderr.puts "ERROR: Failed to load settings file from \"path.settings\". Aborting... path.setting=#{LogStash::SETTINGS.get("path.settings")}, exception=#{e.class}, message=>#{e.message}"
|
156
168
|
return 1
|
157
169
|
end
|
158
170
|
end
|
159
171
|
|
172
|
+
# Configure Logstash logging facility, this need to be done before everything else to
|
173
|
+
# make sure the logger has the correct settings and the log level is correctly defined.
|
174
|
+
# TODO(talevy): cleanly support `path.logs` setting in log4j
|
175
|
+
unless java.lang.System.getProperty("log4j.configurationFile")
|
176
|
+
log4j_config_location = setting("path.settings") + "/log4j2.properties"
|
177
|
+
LogStash::Logging::Logger::initialize(log4j_config_location)
|
178
|
+
end
|
179
|
+
|
160
180
|
super(*[args])
|
161
181
|
end
|
162
182
|
|
@@ -164,12 +184,12 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
164
184
|
require "logstash/util"
|
165
185
|
require "logstash/util/java_version"
|
166
186
|
require "stud/task"
|
167
|
-
require "cabin" # gem 'cabin'
|
168
|
-
require "logstash/logging/json"
|
169
187
|
|
170
|
-
|
171
|
-
|
172
|
-
|
188
|
+
LogStash::Logging::Logger::configure_logging(setting("log.level"))
|
189
|
+
|
190
|
+
if setting("config.debug") && logger.debug?
|
191
|
+
logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
|
192
|
+
end
|
173
193
|
|
174
194
|
LogStash::Util::set_thread_name(self.class.name)
|
175
195
|
|
@@ -186,7 +206,6 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
186
206
|
end
|
187
207
|
|
188
208
|
LogStash::ShutdownWatcher.unsafe_shutdown = setting("pipeline.unsafe_shutdown")
|
189
|
-
LogStash::ShutdownWatcher.logger = @logger
|
190
209
|
|
191
210
|
configure_plugin_paths(setting("path.plugins"))
|
192
211
|
|
@@ -197,7 +216,7 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
197
216
|
|
198
217
|
return start_shell(setting("interactive"), binding) if setting("interactive")
|
199
218
|
|
200
|
-
@settings.format_settings.each {|line|
|
219
|
+
@settings.format_settings.each {|line| logger.info(line) }
|
201
220
|
|
202
221
|
if setting("config.string").nil? && setting("path.config").nil?
|
203
222
|
fail(I18n.t("logstash.runner.missing-configuration"))
|
@@ -209,14 +228,14 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
209
228
|
end
|
210
229
|
|
211
230
|
if setting("config.test_and_exit")
|
212
|
-
config_loader = LogStash::Config::Loader.new(
|
231
|
+
config_loader = LogStash::Config::Loader.new(logger)
|
213
232
|
config_str = config_loader.format_config(setting("path.config"), setting("config.string"))
|
214
233
|
begin
|
215
234
|
LogStash::Pipeline.new(config_str)
|
216
|
-
|
235
|
+
puts "Configuration OK"
|
217
236
|
return 0
|
218
237
|
rescue => e
|
219
|
-
|
238
|
+
logger.fatal I18n.t("logstash.runner.invalid-configuration", :error => e.message)
|
220
239
|
return 1
|
221
240
|
end
|
222
241
|
end
|
@@ -239,6 +258,9 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
239
258
|
|
240
259
|
@agent.shutdown
|
241
260
|
|
261
|
+
# flush any outstanding log messages during shutdown
|
262
|
+
org.apache.logging.log4j.LogManager.shutdown
|
263
|
+
|
242
264
|
agent_return
|
243
265
|
|
244
266
|
rescue Clamp::UsageError => e
|
@@ -246,7 +268,7 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
246
268
|
show_short_help
|
247
269
|
return 1
|
248
270
|
rescue => e
|
249
|
-
|
271
|
+
logger.fatal(I18n.t("oops"), :error => e, :backtrace => e.backtrace)
|
250
272
|
return 1
|
251
273
|
ensure
|
252
274
|
Stud::untrap("INT", sigint_id) unless sigint_id.nil?
|
@@ -258,10 +280,10 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
258
280
|
def show_version
|
259
281
|
show_version_logstash
|
260
282
|
|
261
|
-
if
|
283
|
+
if logger.is_info_enabled
|
262
284
|
show_version_ruby
|
263
285
|
show_version_java if LogStash::Environment.jruby?
|
264
|
-
show_gems if
|
286
|
+
show_gems if logger.debug?
|
265
287
|
end
|
266
288
|
end # def show_version
|
267
289
|
|
@@ -299,55 +321,6 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
299
321
|
LogStash::Agent.new(*args)
|
300
322
|
end
|
301
323
|
|
302
|
-
# Point logging at a specific path.
|
303
|
-
def configure_logging(path, level)
|
304
|
-
@logger = Cabin::Channel.get(LogStash)
|
305
|
-
# Set with the -v (or -vv...) flag
|
306
|
-
case level
|
307
|
-
when "quiet"
|
308
|
-
@logger.level = :error
|
309
|
-
when "verbose"
|
310
|
-
@logger.level = :info
|
311
|
-
when "debug"
|
312
|
-
@logger.level = :debug
|
313
|
-
else
|
314
|
-
@logger.level = :warn
|
315
|
-
end
|
316
|
-
|
317
|
-
if path
|
318
|
-
# TODO(sissel): Implement file output/rotation in Cabin.
|
319
|
-
# TODO(sissel): Catch exceptions, report sane errors.
|
320
|
-
begin
|
321
|
-
@log_fd.close if @log_fd
|
322
|
-
@log_fd = File.new(path, "a")
|
323
|
-
rescue => e
|
324
|
-
fail(I18n.t("logstash.runner.configuration.log_file_failed",
|
325
|
-
:path => path, :error => e))
|
326
|
-
end
|
327
|
-
|
328
|
-
if setting("log.format") == "json"
|
329
|
-
@logger.subscribe(LogStash::Logging::JSON.new(STDOUT), :level => :fatal)
|
330
|
-
@logger.subscribe(LogStash::Logging::JSON.new(@log_fd))
|
331
|
-
else
|
332
|
-
@logger.subscribe(STDOUT, :level => :fatal)
|
333
|
-
@logger.subscribe(@log_fd)
|
334
|
-
end
|
335
|
-
@logger.terminal "Sending logstash logs to #{path}."
|
336
|
-
else
|
337
|
-
if setting("log.format") == "json"
|
338
|
-
@logger.subscribe(LogStash::Logging::JSON.new(STDOUT))
|
339
|
-
else
|
340
|
-
@logger.subscribe(STDOUT)
|
341
|
-
end
|
342
|
-
end
|
343
|
-
|
344
|
-
if setting("config.debug") && @logger.level != :debug
|
345
|
-
@logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
|
346
|
-
end
|
347
|
-
|
348
|
-
# TODO(sissel): redirect stdout/stderr to the log as well
|
349
|
-
# http://jira.codehaus.org/browse/JRUBY-7003
|
350
|
-
end # def configure_logging
|
351
324
|
|
352
325
|
# Emit a failure message and abort.
|
353
326
|
def fail(message)
|
@@ -376,14 +349,14 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
376
349
|
|
377
350
|
def trap_sighup
|
378
351
|
Stud::trap("HUP") do
|
379
|
-
|
352
|
+
logger.warn(I18n.t("logstash.agent.sighup"))
|
380
353
|
@agent.reload_state!
|
381
354
|
end
|
382
355
|
end
|
383
356
|
|
384
357
|
def trap_sigterm
|
385
358
|
Stud::trap("TERM") do
|
386
|
-
|
359
|
+
logger.warn(I18n.t("logstash.agent.sigterm"))
|
387
360
|
@agent_task.stop!
|
388
361
|
end
|
389
362
|
end
|
@@ -391,11 +364,11 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
391
364
|
def trap_sigint
|
392
365
|
Stud::trap("INT") do
|
393
366
|
if @interrupted_once
|
394
|
-
|
367
|
+
logger.fatal(I18n.t("logstash.agent.forced_sigint"))
|
395
368
|
exit
|
396
369
|
else
|
397
|
-
|
398
|
-
Thread.new(
|
370
|
+
logger.warn(I18n.t("logstash.agent.sigint"))
|
371
|
+
Thread.new(logger) {|lg| sleep 5; lg.warn(I18n.t("logstash.agent.slow_shutdown")) }
|
399
372
|
@interrupted_once = true
|
400
373
|
@agent_task.stop!
|
401
374
|
end
|