logstash-core 5.3.3-java → 5.4.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/gemspec_jars.rb +2 -0
- data/lib/logstash-core/logstash-core.jar +0 -0
- data/lib/logstash-core/version.rb +1 -1
- data/lib/logstash-core_jars.rb +4 -0
- data/lib/logstash/agent.rb +15 -6
- data/lib/logstash/api/modules/base.rb +1 -1
- data/lib/logstash/api/rack_app.rb +1 -1
- data/lib/logstash/config/config_ast.rb +13 -13
- data/lib/logstash/config/mixin.rb +33 -28
- data/lib/logstash/environment.rb +11 -0
- data/lib/logstash/event.rb +56 -0
- data/lib/logstash/event_dispatcher.rb +2 -2
- data/lib/logstash/execution_context.rb +10 -0
- data/lib/logstash/filter_delegator.rb +3 -2
- data/lib/logstash/inputs/base.rb +15 -1
- data/lib/logstash/instrument/collector.rb +1 -1
- data/lib/logstash/instrument/metric.rb +4 -2
- data/lib/logstash/instrument/metric_store.rb +9 -5
- data/lib/logstash/instrument/null_metric.rb +1 -0
- data/lib/logstash/instrument/periodic_poller/cgroup.rb +3 -3
- data/lib/logstash/instrument/periodic_poller/jvm.rb +11 -8
- data/lib/logstash/instrument/periodic_poller/load_average.rb +4 -2
- data/lib/logstash/instrument/wrapped_write_client.rb +59 -0
- data/lib/logstash/java_integration.rb +2 -2
- data/lib/logstash/output_delegator.rb +2 -2
- data/lib/logstash/output_delegator_strategies/legacy.rb +5 -2
- data/lib/logstash/output_delegator_strategies/shared.rb +2 -1
- data/lib/logstash/output_delegator_strategies/single.rb +2 -1
- data/lib/logstash/outputs/base.rb +8 -0
- data/lib/logstash/patches/cabin.rb +1 -1
- data/lib/logstash/patches/stronger_openssl_defaults.rb +1 -1
- data/lib/logstash/pipeline.rb +47 -19
- data/lib/logstash/plugin.rb +3 -1
- data/lib/logstash/plugins/hooks_registry.rb +6 -6
- data/lib/logstash/plugins/registry.rb +2 -2
- data/lib/logstash/queue_factory.rb +7 -5
- data/lib/logstash/runner.rb +15 -1
- data/lib/logstash/settings.rb +14 -2
- data/lib/logstash/string_interpolation.rb +18 -0
- data/lib/logstash/timestamp.rb +27 -0
- data/lib/logstash/util.rb +1 -1
- data/lib/logstash/util/prctl.rb +1 -1
- data/lib/logstash/util/retryable.rb +1 -1
- data/lib/logstash/util/wrapped_acked_queue.rb +53 -22
- data/lib/logstash/util/wrapped_synchronous_queue.rb +51 -33
- data/lib/logstash/version.rb +1 -1
- data/locales/en.yml +4 -2
- data/logstash-core.gemspec +0 -3
- data/spec/api/lib/api/node_stats_spec.rb +2 -1
- data/spec/api/spec_helper.rb +1 -1
- data/spec/logstash/acked_queue_concurrent_stress_spec.rb +291 -0
- data/spec/logstash/agent_spec.rb +24 -0
- data/spec/logstash/config/mixin_spec.rb +11 -2
- data/spec/logstash/event_dispatcher_spec.rb +8 -1
- data/spec/logstash/event_spec.rb +346 -0
- data/spec/logstash/execution_context_spec.rb +13 -0
- data/spec/logstash/filter_delegator_spec.rb +4 -2
- data/spec/logstash/inputs/base_spec.rb +41 -0
- data/spec/logstash/instrument/metric_spec.rb +2 -1
- data/spec/logstash/instrument/metric_store_spec.rb +14 -0
- data/spec/logstash/instrument/namespaced_metric_spec.rb +2 -1
- data/spec/logstash/instrument/periodic_poller/cgroup_spec.rb +1 -1
- data/spec/logstash/instrument/periodic_poller/jvm_spec.rb +35 -0
- data/spec/logstash/instrument/periodic_poller/load_average_spec.rb +1 -5
- data/spec/logstash/instrument/wrapped_write_client_spec.rb +113 -0
- data/spec/logstash/json_spec.rb +1 -1
- data/spec/logstash/legacy_ruby_event_spec.rb +636 -0
- data/spec/logstash/legacy_ruby_timestamp_spec.rb +170 -0
- data/spec/logstash/output_delegator_spec.rb +6 -3
- data/spec/logstash/outputs/base_spec.rb +23 -0
- data/spec/logstash/pipeline_pq_file_spec.rb +18 -8
- data/spec/logstash/pipeline_spec.rb +41 -5
- data/spec/logstash/plugin_spec.rb +15 -3
- data/spec/logstash/plugins/hooks_registry_spec.rb +2 -2
- data/spec/logstash/runner_spec.rb +33 -2
- data/spec/logstash/settings/port_range_spec.rb +1 -1
- data/spec/logstash/settings_spec.rb +21 -0
- data/spec/logstash/timestamp_spec.rb +29 -0
- data/spec/logstash/util/accessors_spec.rb +179 -0
- data/spec/logstash/util/wrapped_synchronous_queue_spec.rb +4 -11
- data/spec/logstash/util_spec.rb +1 -1
- data/spec/logstash/webserver_spec.rb +1 -1
- data/spec/support/mocks_classes.rb +65 -53
- metadata +25 -30
@@ -14,7 +14,7 @@ module LogStash
|
|
14
14
|
]
|
15
15
|
def_delegators :@filter, *DELEGATED_METHODS
|
16
16
|
|
17
|
-
def initialize(logger, klass, metric, plugin_args)
|
17
|
+
def initialize(logger, klass, metric, execution_context, plugin_args)
|
18
18
|
@logger = logger
|
19
19
|
@klass = klass
|
20
20
|
@id = plugin_args["id"]
|
@@ -23,6 +23,7 @@ module LogStash
|
|
23
23
|
# Scope the metrics to the plugin
|
24
24
|
namespaced_metric = metric.namespace(@id.to_sym)
|
25
25
|
@filter.metric = namespaced_metric
|
26
|
+
@filter.execution_context = execution_context
|
26
27
|
|
27
28
|
@metric_events = namespaced_metric.namespace(:events)
|
28
29
|
namespaced_metric.gauge(:name, config_name)
|
@@ -42,7 +43,7 @@ module LogStash
|
|
42
43
|
new_events = @filter.multi_filter(events)
|
43
44
|
clock.stop
|
44
45
|
|
45
|
-
# There is no
|
46
|
+
# There is no guarantee in the context of filter
|
46
47
|
# that EVENTS_INT == EVENTS_OUT, see the aggregates and
|
47
48
|
# the split filter
|
48
49
|
c = new_events.count { |event| !event.cancelled? }
|
data/lib/logstash/inputs/base.rb
CHANGED
@@ -89,11 +89,25 @@ class LogStash::Inputs::Base < LogStash::Plugin
|
|
89
89
|
stop
|
90
90
|
end
|
91
91
|
|
92
|
-
# stop? should never be
|
92
|
+
# stop? should never be overridden
|
93
93
|
public
|
94
94
|
def stop?
|
95
95
|
@stop_called.value
|
96
96
|
end
|
97
|
+
|
98
|
+
def clone
|
99
|
+
cloned = super
|
100
|
+
cloned.codec = @codec.clone if @codec
|
101
|
+
cloned
|
102
|
+
end
|
103
|
+
|
104
|
+
def execution_context=(context)
|
105
|
+
super
|
106
|
+
# There is no easy way to propage an instance variable into the codec, because the codec
|
107
|
+
# are created at the class level
|
108
|
+
@codec.execution_context = context
|
109
|
+
context
|
110
|
+
end
|
97
111
|
|
98
112
|
protected
|
99
113
|
def decorate(event)
|
@@ -29,7 +29,7 @@ module LogStash module Instrument
|
|
29
29
|
# of update the metric
|
30
30
|
#
|
31
31
|
# If there is a problem with the key or the type of metric we will record an error
|
32
|
-
# but we
|
32
|
+
# but we won't stop processing events, theses errors are not considered fatal.
|
33
33
|
#
|
34
34
|
def push(namespaces_path, key, type, *metric_type_params)
|
35
35
|
begin
|
@@ -77,7 +77,7 @@ module LogStash module Instrument
|
|
77
77
|
private
|
78
78
|
# Allow to calculate the execution of a block of code.
|
79
79
|
# This class support 2 differents syntax a block or the return of
|
80
|
-
# the object itself, but in the later case the metric
|
80
|
+
# the object itself, but in the later case the metric won't be recorded
|
81
81
|
# Until we call `#stop`.
|
82
82
|
#
|
83
83
|
# @see LogStash::Instrument::Metric#time
|
@@ -96,7 +96,9 @@ module LogStash module Instrument
|
|
96
96
|
end
|
97
97
|
|
98
98
|
def stop
|
99
|
-
|
99
|
+
execution_time = (MILLISECONDS * (Time.now - @start_time)).to_i
|
100
|
+
@metric.report_time(@namespace, @key, execution_time)
|
101
|
+
execution_time
|
100
102
|
end
|
101
103
|
end
|
102
104
|
end
|
@@ -52,7 +52,7 @@ module LogStash module Instrument
|
|
52
52
|
# BUT. If the value is not present in the `@fast_lookup` the value will be inserted and
|
53
53
|
# `#puf_if_absent` will return nil. With this returned value of nil we assume that we don't
|
54
54
|
# have it in the `@metric_store` for structured search so we add it there too.
|
55
|
-
if found_value = @fast_lookup.put_if_absent(
|
55
|
+
if found_value = @fast_lookup.put_if_absent(namespaces.dup << key, provided_value)
|
56
56
|
return found_value
|
57
57
|
else
|
58
58
|
@structured_lookup_mutex.synchronize do
|
@@ -73,7 +73,7 @@ module LogStash module Instrument
|
|
73
73
|
# If you use the `,` on a key the metric store will return the both values at that level
|
74
74
|
#
|
75
75
|
# The returned hash will keep the same structure as it had in the `Concurrent::Map`
|
76
|
-
# but will be a normal ruby hash. This will allow the api to easily
|
76
|
+
# but will be a normal ruby hash. This will allow the api to easily serialize the content
|
77
77
|
# of the map
|
78
78
|
#
|
79
79
|
# @param [Array] The path where values should be located
|
@@ -131,7 +131,7 @@ module LogStash module Instrument
|
|
131
131
|
# }
|
132
132
|
def extract_metrics(path, *keys)
|
133
133
|
keys.reduce({}) do |acc,k|
|
134
|
-
#
|
134
|
+
# Simplify 1-length keys
|
135
135
|
k = k.first if k.is_a?(Array) && k.size == 1
|
136
136
|
|
137
137
|
# If we have array values here we need to recurse
|
@@ -162,6 +162,10 @@ module LogStash module Instrument
|
|
162
162
|
end
|
163
163
|
end
|
164
164
|
|
165
|
+
def has_metric?(*path)
|
166
|
+
@fast_lookup[path]
|
167
|
+
end
|
168
|
+
|
165
169
|
# Return all the individuals Metric,
|
166
170
|
# This call mimic a Enum's each if a block is provided
|
167
171
|
#
|
@@ -179,9 +183,9 @@ module LogStash module Instrument
|
|
179
183
|
alias_method :all, :each
|
180
184
|
|
181
185
|
def prune(path)
|
182
|
-
key_paths = key_paths(path).map
|
186
|
+
key_paths = key_paths(path).map(&:to_sym)
|
183
187
|
@structured_lookup_mutex.synchronize do
|
184
|
-
keys_to_delete = @fast_lookup.keys.select {|namespace
|
188
|
+
keys_to_delete = @fast_lookup.keys.select {|namespace| (key_paths - namespace[0..-2]).empty? }
|
185
189
|
keys_to_delete.each {|k| @fast_lookup.delete(k) }
|
186
190
|
delete_from_map(@store, key_paths)
|
187
191
|
end
|
@@ -9,7 +9,7 @@ module LogStash module Instrument module PeriodicPoller
|
|
9
9
|
include LogStash::Util::Loggable
|
10
10
|
|
11
11
|
CONTROL_GROUP_RE = Regexp.compile("\\d+:([^:,]+(?:,[^:,]+)?):(/.*)");
|
12
|
-
|
12
|
+
CONTROLLER_SEPARATOR_RE = ","
|
13
13
|
|
14
14
|
PROC_SELF_CGROUP_FILE = Pathname.new("/proc/self/cgroup")
|
15
15
|
PROC_CGROUP_CPU_DIR = Pathname.new("/sys/fs/cgroup/cpu")
|
@@ -36,8 +36,8 @@ module LogStash module Instrument module PeriodicPoller
|
|
36
36
|
|
37
37
|
read_proc_self_cgroup_lines.each do |line|
|
38
38
|
matches = CONTROL_GROUP_RE.match(line)
|
39
|
-
# multiples
|
40
|
-
controllers = matches[1].split(
|
39
|
+
# multiples controls, same hierarchy
|
40
|
+
controllers = matches[1].split(CONTROLLER_SEPARATOR_RE)
|
41
41
|
controllers.each_with_object(response) { |controller| response[controller] = matches[2] }
|
42
42
|
end
|
43
43
|
|
@@ -34,6 +34,13 @@ module LogStash module Instrument module PeriodicPoller
|
|
34
34
|
end
|
35
35
|
end
|
36
36
|
|
37
|
+
MEMORY_TRANSPOSE_MAP = {
|
38
|
+
"usage.used" => :used_in_bytes,
|
39
|
+
"usage.committed" => :committed_in_bytes,
|
40
|
+
"usage.max" => :max_in_bytes,
|
41
|
+
"peak.max" => :peak_max_in_bytes,
|
42
|
+
"peak.used" => :peak_used_in_bytes
|
43
|
+
}
|
37
44
|
|
38
45
|
attr_reader :metric
|
39
46
|
|
@@ -52,8 +59,6 @@ module LogStash module Instrument module PeriodicPoller
|
|
52
59
|
collect_load_average
|
53
60
|
end
|
54
61
|
|
55
|
-
private
|
56
|
-
|
57
62
|
def collect_gc_stats
|
58
63
|
garbage_collectors = ManagementFactory.getGarbageCollectorMXBeans()
|
59
64
|
|
@@ -141,7 +146,6 @@ module LogStash module Instrument module PeriodicPoller
|
|
141
146
|
end
|
142
147
|
end
|
143
148
|
|
144
|
-
|
145
149
|
def build_pools_metrics(data)
|
146
150
|
heap = data["heap"]
|
147
151
|
old = {}
|
@@ -164,11 +168,10 @@ module LogStash module Instrument module PeriodicPoller
|
|
164
168
|
collection.reduce(default_information_accumulator) do |m,e|
|
165
169
|
e = { e[0] => e[1] } if e.is_a?(Array)
|
166
170
|
e.each_pair do |k,v|
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
m[:peak_used_in_bytes] += v if k.include?("peak.used")
|
171
|
+
if MEMORY_TRANSPOSE_MAP.include?(k)
|
172
|
+
transpose_key = MEMORY_TRANSPOSE_MAP[k]
|
173
|
+
m[transpose_key] += v
|
174
|
+
end
|
172
175
|
end
|
173
176
|
m
|
174
177
|
end
|
@@ -1,4 +1,6 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
+
java_import "java.lang.management.ManagementFactory"
|
3
|
+
|
2
4
|
module LogStash module Instrument module PeriodicPoller
|
3
5
|
class LoadAverage
|
4
6
|
class Windows
|
@@ -11,8 +13,8 @@ module LogStash module Instrument module PeriodicPoller
|
|
11
13
|
LOAD_AVG_FILE = "/proc/loadavg"
|
12
14
|
TOKEN_SEPARATOR = " "
|
13
15
|
|
14
|
-
def self.get
|
15
|
-
load_average =
|
16
|
+
def self.get(content = ::File.read(LOAD_AVG_FILE))
|
17
|
+
load_average = content.chomp.split(TOKEN_SEPARATOR)
|
16
18
|
|
17
19
|
{
|
18
20
|
:"1m" => load_average[0].to_f,
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
module LogStash module Instrument
|
3
|
+
class WrappedWriteClient
|
4
|
+
def initialize(write_client, pipeline, metric, plugin)
|
5
|
+
@write_client = write_client
|
6
|
+
|
7
|
+
pipeline_id = pipeline.pipeline_id.to_s.to_sym
|
8
|
+
plugin_type = "#{plugin.class.plugin_type}s".to_sym
|
9
|
+
|
10
|
+
@events_metrics = metric.namespace([:stats, :events])
|
11
|
+
@pipeline_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :events])
|
12
|
+
@plugin_events_metrics = metric.namespace([:stats, :pipelines, pipeline_id, :plugins, plugin_type, plugin.id.to_sym, :events])
|
13
|
+
|
14
|
+
define_initial_metrics_values
|
15
|
+
end
|
16
|
+
|
17
|
+
def get_new_batch
|
18
|
+
@write_client.get_new_batch
|
19
|
+
end
|
20
|
+
|
21
|
+
def push(event)
|
22
|
+
record_metric { @write_client.push(event) }
|
23
|
+
end
|
24
|
+
alias_method(:<<, :push)
|
25
|
+
|
26
|
+
def push_batch(batch)
|
27
|
+
record_metric(batch.size) { @write_client.push_batch(batch) }
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
def record_metric(size = 1)
|
32
|
+
@events_metrics.increment(:in, size)
|
33
|
+
@pipeline_metrics.increment(:in, size)
|
34
|
+
@plugin_events_metrics.increment(:out, size)
|
35
|
+
|
36
|
+
clock = @events_metrics.time(:queue_push_duration_in_millis)
|
37
|
+
|
38
|
+
result = yield
|
39
|
+
|
40
|
+
# Reuse the same values for all the endpoints to make sure we don't have skew in times.
|
41
|
+
execution_time = clock.stop
|
42
|
+
|
43
|
+
@pipeline_metrics.report_time(:queue_push_duration_in_millis, execution_time)
|
44
|
+
@plugin_events_metrics.report_time(:queue_push_duration_in_millis, execution_time)
|
45
|
+
|
46
|
+
result
|
47
|
+
end
|
48
|
+
|
49
|
+
def define_initial_metrics_values
|
50
|
+
@events_metrics.increment(:in, 0)
|
51
|
+
@pipeline_metrics.increment(:in, 0)
|
52
|
+
@plugin_events_metrics.increment(:out, 0)
|
53
|
+
|
54
|
+
@events_metrics.report_time(:queue_push_duration_in_millis, 0)
|
55
|
+
@pipeline_metrics.report_time(:queue_push_duration_in_millis, 0)
|
56
|
+
@plugin_events_metrics.report_time(:queue_push_duration_in_millis, 0)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "java"
|
3
3
|
|
4
|
-
# this is mainly for usage with JrJackson json parsing in :raw mode which
|
4
|
+
# this is mainly for usage with JrJackson json parsing in :raw mode which generates
|
5
5
|
# Java::JavaUtil::ArrayList and Java::JavaUtil::LinkedHashMap native objects for speed.
|
6
6
|
# these object already quacks like their Ruby equivalents Array and Hash but they will
|
7
7
|
# not test for is_a?(Array) or is_a?(Hash) and we do not want to include tests for
|
@@ -35,7 +35,7 @@ map_mixin = lambda do
|
|
35
35
|
# this bug makes has_key? (and all its aliases) return false for a key that has a nil value.
|
36
36
|
# Only LinkedHashMap is patched here because patching the Map interface is not working.
|
37
37
|
# TODO find proper fix, and submit upstream
|
38
|
-
#
|
38
|
+
# relevant JRuby files:
|
39
39
|
# https://github.com/jruby/jruby/blob/master/core/src/main/ruby/jruby/java/java_ext/java.util.rb
|
40
40
|
# https://github.com/jruby/jruby/blob/master/core/src/main/java/org/jruby/java/proxies/MapJavaProxy.java
|
41
41
|
def has_key?(key)
|
@@ -7,7 +7,7 @@ require "logstash/output_delegator_strategies/legacy"
|
|
7
7
|
module LogStash class OutputDelegator
|
8
8
|
attr_reader :metric, :metric_events, :strategy, :namespaced_metric, :metric_events, :id
|
9
9
|
|
10
|
-
def initialize(logger, output_class, metric, strategy_registry, plugin_args)
|
10
|
+
def initialize(logger, output_class, metric, execution_context, strategy_registry, plugin_args)
|
11
11
|
@logger = logger
|
12
12
|
@output_class = output_class
|
13
13
|
@metric = metric
|
@@ -22,7 +22,7 @@ module LogStash class OutputDelegator
|
|
22
22
|
|
23
23
|
@strategy = strategy_registry.
|
24
24
|
class_for(self.concurrency).
|
25
|
-
new(@logger, @output_class, @namespaced_metric, plugin_args)
|
25
|
+
new(@logger, @output_class, @namespaced_metric, execution_context, plugin_args)
|
26
26
|
end
|
27
27
|
|
28
28
|
def config_name
|
@@ -2,10 +2,13 @@
|
|
2
2
|
module LogStash module OutputDelegatorStrategies class Legacy
|
3
3
|
attr_reader :worker_count, :workers
|
4
4
|
|
5
|
-
def initialize(logger, klass, metric, plugin_args)
|
5
|
+
def initialize(logger, klass, metric, execution_context, plugin_args)
|
6
6
|
@worker_count = (plugin_args["workers"] || 1).to_i
|
7
7
|
@workers = @worker_count.times.map { klass.new(plugin_args) }
|
8
|
-
@workers.each
|
8
|
+
@workers.each do |w|
|
9
|
+
w.metric = metric
|
10
|
+
w.execution_context = execution_context
|
11
|
+
end
|
9
12
|
@worker_queue = SizedQueue.new(@worker_count)
|
10
13
|
@workers.each {|w| @worker_queue << w}
|
11
14
|
end
|
@@ -1,7 +1,8 @@
|
|
1
1
|
module LogStash module OutputDelegatorStrategies class Shared
|
2
|
-
def initialize(logger, klass, metric, plugin_args)
|
2
|
+
def initialize(logger, klass, metric, execution_context, plugin_args)
|
3
3
|
@output = klass.new(plugin_args)
|
4
4
|
@output.metric = metric
|
5
|
+
@output.execution_context = execution_context
|
5
6
|
end
|
6
7
|
|
7
8
|
def register
|
@@ -1,7 +1,8 @@
|
|
1
1
|
module LogStash module OutputDelegatorStrategies class Single
|
2
|
-
def initialize(logger, klass, metric, plugin_args)
|
2
|
+
def initialize(logger, klass, metric, execution_context, plugin_args)
|
3
3
|
@output = klass.new(plugin_args)
|
4
4
|
@output.metric = metric
|
5
|
+
@output.execution_context = execution_context
|
5
6
|
@mutex = Mutex.new
|
6
7
|
end
|
7
8
|
|
@@ -105,6 +105,14 @@ class LogStash::Outputs::Base < LogStash::Plugin
|
|
105
105
|
self.class.concurrency
|
106
106
|
end
|
107
107
|
|
108
|
+
def execution_context=(context)
|
109
|
+
super
|
110
|
+
# There is no easy way to propage an instance variable into the codec, because the codec
|
111
|
+
# are created at the class level
|
112
|
+
@codec.execution_context = context
|
113
|
+
context
|
114
|
+
end
|
115
|
+
|
108
116
|
private
|
109
117
|
def output?(event)
|
110
118
|
# TODO: noop for now, remove this once we delete this call from all plugins
|
@@ -9,7 +9,7 @@ if ENV["PROFILE_BAD_LOG_CALLS"] || ($DEBUGLIST || []).include?("log")
|
|
9
9
|
# Basically, the following is wastes tons of effort creating objects that are
|
10
10
|
# never used if the log level hides the log:
|
11
11
|
#
|
12
|
-
# logger.debug("something
|
12
|
+
# logger.debug("something happened", :what => Happened)
|
13
13
|
#
|
14
14
|
# This is shown to be 4x faster:
|
15
15
|
#
|
@@ -54,7 +54,7 @@ class OpenSSL::SSL::SSLContext
|
|
54
54
|
#
|
55
55
|
# This monkeypatch doesn't enforce a `VERIFY_MODE` on the SSLContext,
|
56
56
|
# SSLContext are both used for the client and the server implementation,
|
57
|
-
# If set the `verify_mode` to peer the server
|
57
|
+
# If set the `verify_mode` to peer the server won't accept any connection,
|
58
58
|
# because it will try to verify the client certificate, this is a protocol
|
59
59
|
# details implemented at the plugin level.
|
60
60
|
#
|
data/lib/logstash/pipeline.rb
CHANGED
@@ -17,9 +17,11 @@ require "logstash/instrument/namespaced_metric"
|
|
17
17
|
require "logstash/instrument/null_metric"
|
18
18
|
require "logstash/instrument/namespaced_null_metric"
|
19
19
|
require "logstash/instrument/collector"
|
20
|
+
require "logstash/instrument/wrapped_write_client"
|
20
21
|
require "logstash/output_delegator"
|
21
22
|
require "logstash/filter_delegator"
|
22
23
|
require "logstash/queue_factory"
|
24
|
+
require "logstash/execution_context"
|
23
25
|
|
24
26
|
module LogStash; class BasePipeline
|
25
27
|
include LogStash::Util::Loggable
|
@@ -41,6 +43,7 @@ module LogStash; class BasePipeline
|
|
41
43
|
@inputs = nil
|
42
44
|
@filters = nil
|
43
45
|
@outputs = nil
|
46
|
+
@execution_context = LogStash::ExecutionContext.new(@pipeline_id)
|
44
47
|
|
45
48
|
grammar = LogStashConfigParser.new
|
46
49
|
parsed_config = grammar.parse(config_str)
|
@@ -88,12 +91,15 @@ module LogStash; class BasePipeline
|
|
88
91
|
klass = Plugin.lookup(plugin_type, name)
|
89
92
|
|
90
93
|
if plugin_type == "output"
|
91
|
-
OutputDelegator.new(@logger, klass, type_scoped_metric,
|
94
|
+
OutputDelegator.new(@logger, klass, type_scoped_metric, @execution_context, OutputDelegatorStrategyRegistry.instance, args)
|
92
95
|
elsif plugin_type == "filter"
|
93
|
-
FilterDelegator.new(@logger, klass, type_scoped_metric, args)
|
96
|
+
FilterDelegator.new(@logger, klass, type_scoped_metric, @execution_context, args)
|
94
97
|
else # input
|
95
98
|
input_plugin = klass.new(args)
|
96
|
-
|
99
|
+
scoped_metric = type_scoped_metric.namespace(id.to_sym)
|
100
|
+
scoped_metric.gauge(:name, input_plugin.config_name)
|
101
|
+
input_plugin.metric = scoped_metric
|
102
|
+
input_plugin.execution_context = @execution_context
|
97
103
|
input_plugin
|
98
104
|
end
|
99
105
|
end
|
@@ -148,12 +154,13 @@ module LogStash; class Pipeline < BasePipeline
|
|
148
154
|
@input_queue_client = @queue.write_client
|
149
155
|
@filter_queue_client = @queue.read_client
|
150
156
|
@signal_queue = Queue.new
|
151
|
-
# Note that @
|
157
|
+
# Note that @inflight_batches as a central mechanism for tracking inflight
|
152
158
|
# batches will fail if we have multiple read clients here.
|
153
159
|
@filter_queue_client.set_events_metric(metric.namespace([:stats, :events]))
|
154
160
|
@filter_queue_client.set_pipeline_metric(
|
155
161
|
metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :events])
|
156
162
|
)
|
163
|
+
@drain_queue = @settings.get_value("queue.drain")
|
157
164
|
|
158
165
|
@events_filtered = Concurrent::AtomicFixnum.new(0)
|
159
166
|
@events_consumed = Concurrent::AtomicFixnum.new(0)
|
@@ -250,6 +257,10 @@ module LogStash; class Pipeline < BasePipeline
|
|
250
257
|
@running.false?
|
251
258
|
end
|
252
259
|
|
260
|
+
def system?
|
261
|
+
settings.get_value("pipeline.system")
|
262
|
+
end
|
263
|
+
|
253
264
|
# register_plugin simply calls the plugin #register method and catches & logs any error
|
254
265
|
# @param plugin [Plugin] the plugin to register
|
255
266
|
# @return [Plugin] the registered plugin
|
@@ -326,26 +337,32 @@ module LogStash; class Pipeline < BasePipeline
|
|
326
337
|
# Main body of what a worker thread does
|
327
338
|
# Repeatedly takes batches off the queue, filters, then outputs them
|
328
339
|
def worker_loop(batch_size, batch_delay)
|
329
|
-
|
340
|
+
shutdown_requested = false
|
330
341
|
|
331
342
|
@filter_queue_client.set_batch_dimensions(batch_size, batch_delay)
|
332
343
|
|
333
|
-
while
|
334
|
-
batch = @filter_queue_client.take_batch
|
344
|
+
while true
|
335
345
|
signal = @signal_queue.empty? ? NO_SIGNAL : @signal_queue.pop
|
336
|
-
|
346
|
+
shutdown_requested |= signal.shutdown? # latch on shutdown signal
|
337
347
|
|
348
|
+
batch = @filter_queue_client.read_batch # metrics are started in read_batch
|
338
349
|
@events_consumed.increment(batch.size)
|
339
|
-
|
340
350
|
filter_batch(batch)
|
341
|
-
|
342
|
-
if signal.flush? || signal.shutdown?
|
343
|
-
flush_filters_to_batch(batch, :final => signal.shutdown?)
|
344
|
-
end
|
345
|
-
|
351
|
+
flush_filters_to_batch(batch, :final => false) if signal.flush?
|
346
352
|
output_batch(batch)
|
347
353
|
@filter_queue_client.close_batch(batch)
|
354
|
+
|
355
|
+
# keep break at end of loop, after the read_batch operation, some pipeline specs rely on this "final read_batch" before shutdown.
|
356
|
+
break if shutdown_requested && !draining_queue?
|
348
357
|
end
|
358
|
+
|
359
|
+
# we are shutting down, queue is drained if it was required, now perform a final flush.
|
360
|
+
# for this we need to create a new empty batch to contain the final flushed events
|
361
|
+
batch = @filter_queue_client.new_batch
|
362
|
+
@filter_queue_client.start_metrics(batch) # explicitly call start_metrics since we dont do a read_batch here
|
363
|
+
flush_filters_to_batch(batch, :final => true)
|
364
|
+
output_batch(batch)
|
365
|
+
@filter_queue_client.close_batch(batch)
|
349
366
|
end
|
350
367
|
|
351
368
|
def filter_batch(batch)
|
@@ -411,7 +428,7 @@ module LogStash; class Pipeline < BasePipeline
|
|
411
428
|
# first make sure we can register all input plugins
|
412
429
|
register_plugins(@inputs)
|
413
430
|
|
414
|
-
# then after all input plugins are
|
431
|
+
# then after all input plugins are successfully registered, start them
|
415
432
|
@inputs.each { |input| start_input(input) }
|
416
433
|
end
|
417
434
|
|
@@ -422,7 +439,8 @@ module LogStash; class Pipeline < BasePipeline
|
|
422
439
|
def inputworker(plugin)
|
423
440
|
Util::set_thread_name("[#{pipeline_id}]<#{plugin.class.config_name}")
|
424
441
|
begin
|
425
|
-
plugin
|
442
|
+
input_queue_client = wrapped_write_client(plugin)
|
443
|
+
plugin.run(input_queue_client)
|
426
444
|
rescue => e
|
427
445
|
if plugin.stop?
|
428
446
|
@logger.debug("Input plugin raised exception during shutdown, ignoring it.",
|
@@ -456,11 +474,11 @@ module LogStash; class Pipeline < BasePipeline
|
|
456
474
|
# @param before_stop [Proc] code block called before performing stop operation on input plugins
|
457
475
|
def shutdown(&before_stop)
|
458
476
|
# shutdown can only start once the pipeline has completed its startup.
|
459
|
-
# avoid potential race
|
477
|
+
# avoid potential race condition between the startup sequence and this
|
460
478
|
# shutdown method which can be called from another thread at any time
|
461
479
|
sleep(0.1) while !ready?
|
462
480
|
|
463
|
-
# TODO: should we also check against calling shutdown multiple times
|
481
|
+
# TODO: should we also check against calling shutdown multiple times concurrently?
|
464
482
|
|
465
483
|
before_stop.call if block_given?
|
466
484
|
|
@@ -604,4 +622,14 @@ module LogStash; class Pipeline < BasePipeline
|
|
604
622
|
:flushing => @flushing
|
605
623
|
}
|
606
624
|
end
|
607
|
-
|
625
|
+
|
626
|
+
private
|
627
|
+
|
628
|
+
def draining_queue?
|
629
|
+
@drain_queue ? !@filter_queue_client.empty? : false
|
630
|
+
end
|
631
|
+
|
632
|
+
def wrapped_write_client(plugin)
|
633
|
+
LogStash::Instrument::WrappedWriteClient.new(@input_queue_client, self, metric, plugin)
|
634
|
+
end
|
635
|
+
end; end
|