semantic_logger 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/api.md +160 -9
- data/docs/appenders.md +461 -34
- data/docs/config.md +543 -36
- data/docs/log.md +38 -7
- data/docs/metrics.md +139 -28
- data/docs/operations.md +293 -24
- data/docs/rails.md +319 -5
- data/docs/security.md +135 -107
- data/docs/testing.md +23 -0
- data/docs/upgrading.md +644 -1
- data/lib/semantic_logger/appender/async.rb +29 -11
- data/lib/semantic_logger/appender/bugsnag.rb +7 -2
- data/lib/semantic_logger/appender/cloudwatch_logs.rb +46 -6
- data/lib/semantic_logger/appender/elasticsearch_base.rb +197 -29
- data/lib/semantic_logger/appender/elasticsearch_http.rb +22 -5
- data/lib/semantic_logger/appender/file.rb +29 -14
- data/lib/semantic_logger/appender/graylog.rb +81 -7
- data/lib/semantic_logger/appender/honeybadger.rb +7 -1
- data/lib/semantic_logger/appender/honeybadger_insights.rb +8 -2
- data/lib/semantic_logger/appender/http.rb +244 -41
- data/lib/semantic_logger/appender/io.rb +11 -6
- data/lib/semantic_logger/appender/kafka.rb +192 -46
- data/lib/semantic_logger/appender/loki.rb +42 -6
- data/lib/semantic_logger/appender/mongodb.rb +94 -9
- data/lib/semantic_logger/appender/new_relic.rb +2 -3
- data/lib/semantic_logger/appender/new_relic_logs.rb +30 -8
- data/lib/semantic_logger/appender/open_telemetry.rb +37 -35
- data/lib/semantic_logger/appender/rabbitmq.rb +241 -17
- data/lib/semantic_logger/appender/sentry.rb +2 -3
- data/lib/semantic_logger/appender/sentry_ruby.rb +82 -15
- data/lib/semantic_logger/appender/splunk.rb +4 -4
- data/lib/semantic_logger/appender/splunk_http.rb +92 -1
- data/lib/semantic_logger/appender/syslog.rb +89 -10
- data/lib/semantic_logger/appender/tcp.rb +68 -98
- data/lib/semantic_logger/appender/text_output.rb +65 -0
- data/lib/semantic_logger/appender/udp.rb +130 -18
- data/lib/semantic_logger/appender/wrapper.rb +10 -7
- data/lib/semantic_logger/appender.rb +115 -10
- data/lib/semantic_logger/appenders.rb +136 -4
- data/lib/semantic_logger/backtrace_cleaning.rb +77 -0
- data/lib/semantic_logger/base.rb +53 -71
- data/lib/semantic_logger/call_site.rb +235 -0
- data/lib/semantic_logger/context.rb +222 -0
- data/lib/semantic_logger/core_ext/process.rb +20 -5
- data/lib/semantic_logger/core_ext/warning.rb +70 -0
- data/lib/semantic_logger/deprecation.rb +139 -0
- data/lib/semantic_logger/exception_details.rb +4 -3
- data/lib/semantic_logger/formatters/base.rb +139 -19
- data/lib/semantic_logger/formatters/color.rb +3 -4
- data/lib/semantic_logger/formatters/default.rb +35 -7
- data/lib/semantic_logger/formatters/ecs.rb +9 -4
- data/lib/semantic_logger/formatters/json.rb +9 -0
- data/lib/semantic_logger/formatters/logfmt.rb +9 -7
- data/lib/semantic_logger/formatters/loki/labels.rb +282 -0
- data/lib/semantic_logger/formatters/loki.rb +412 -24
- data/lib/semantic_logger/formatters/new_relic_logs.rb +15 -13
- data/lib/semantic_logger/formatters/one_line.rb +44 -10
- data/lib/semantic_logger/formatters/open_telemetry.rb +186 -29
- data/lib/semantic_logger/formatters/pattern.rb +60 -8
- data/lib/semantic_logger/formatters/raw.rb +38 -8
- data/lib/semantic_logger/formatters/signalfx.rb +55 -15
- data/lib/semantic_logger/formatters/syslog.rb +4 -2
- data/lib/semantic_logger/formatters/syslog_cee.rb +10 -4
- data/lib/semantic_logger/formatters/syslog_header.rb +49 -0
- data/lib/semantic_logger/formatters.rb +1 -0
- data/lib/semantic_logger/log.rb +236 -80
- data/lib/semantic_logger/loggable.rb +55 -95
- data/lib/semantic_logger/logger.rb +31 -5
- data/lib/semantic_logger/metric/new_relic.rb +32 -17
- data/lib/semantic_logger/metric/signalfx.rb +21 -12
- data/lib/semantic_logger/metric/statsd.rb +26 -8
- data/lib/semantic_logger/processor.rb +194 -5
- data/lib/semantic_logger/queue_processor/backoff.rb +140 -0
- data/lib/semantic_logger/queue_processor/batch_failure.rb +69 -0
- data/lib/semantic_logger/queue_processor/held_write.rb +62 -0
- data/lib/semantic_logger/queue_processor/inbox.rb +97 -0
- data/lib/semantic_logger/queue_processor/reopener.rb +56 -0
- data/lib/semantic_logger/queue_processor/reporter.rb +113 -0
- data/lib/semantic_logger/queue_processor/request.rb +48 -10
- data/lib/semantic_logger/queue_processor/throttled_counter.rb +60 -17
- data/lib/semantic_logger/queue_processor.rb +426 -176
- data/lib/semantic_logger/ractor_copy.rb +120 -0
- data/lib/semantic_logger/ractor_processor.rb +82 -0
- data/lib/semantic_logger/ractor_receiver.rb +282 -0
- data/lib/semantic_logger/ractor_support.rb +28 -0
- data/lib/semantic_logger/redactor.rb +310 -0
- data/lib/semantic_logger/semantic_logger.rb +313 -79
- data/lib/semantic_logger/subscriber.rb +169 -15
- data/lib/semantic_logger/sync_processor.rb +226 -26
- data/lib/semantic_logger/test/capture_log_events.rb +20 -2
- data/lib/semantic_logger/test/minitest.rb +2 -1
- data/lib/semantic_logger/test/rspec.rb +1 -1
- data/lib/semantic_logger/trap_safe.rb +113 -0
- data/lib/semantic_logger/utils.rb +266 -79
- data/lib/semantic_logger/version.rb +1 -1
- data/lib/semantic_logger.rb +35 -2
- metadata +22 -2
|
@@ -12,14 +12,37 @@ module SemanticLogger
|
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
subscribers = (@subscribers ||= Concurrent::Array.new)
|
|
15
|
-
|
|
15
|
+
return if subscribers.include?(subscriber)
|
|
16
|
+
|
|
17
|
+
subscribers << subscriber
|
|
18
|
+
# A Ractor other than the main one can only call a shareable subscriber.
|
|
19
|
+
if RactorSupport.enabled? && ::Ractor.shareable?(subscriber)
|
|
20
|
+
@ractor_subscribers = [*@ractor_subscribers, subscriber].freeze
|
|
21
|
+
end
|
|
22
|
+
subscribers
|
|
16
23
|
end
|
|
17
24
|
|
|
18
25
|
class << self
|
|
19
26
|
attr_reader :subscribers
|
|
20
27
|
end
|
|
21
28
|
|
|
29
|
+
# Removes the on_log subscriber.
|
|
30
|
+
def self.unsubscribe(subscriber)
|
|
31
|
+
@subscribers&.delete(subscriber)
|
|
32
|
+
@ractor_subscribers = (@ractor_subscribers - [subscriber]).freeze if @ractor_subscribers
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Removes every on_log subscriber.
|
|
36
|
+
def self.clear_subscribers
|
|
37
|
+
@subscribers&.clear
|
|
38
|
+
@ractor_subscribers = nil
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Returns the processor that writes log entries to the appenders, or in a Ractor other than the
|
|
42
|
+
# main one, the RactorProcessor that sends them to the main Ractor.
|
|
22
43
|
def self.processor
|
|
44
|
+
return RactorProcessor.current if RactorSupport.other_ractor?
|
|
45
|
+
|
|
23
46
|
@processor ||= Processor.new
|
|
24
47
|
end
|
|
25
48
|
|
|
@@ -74,13 +97,16 @@ module SemanticLogger
|
|
|
74
97
|
Logger.processor.log(log)
|
|
75
98
|
end
|
|
76
99
|
|
|
77
|
-
@processor
|
|
78
|
-
@subscribers
|
|
100
|
+
@processor = nil
|
|
101
|
+
@subscribers = nil
|
|
102
|
+
@ractor_subscribers = nil
|
|
79
103
|
|
|
104
|
+
# Calls the on_log subscribers, or in a Ractor other than the main one, the shareable ones.
|
|
80
105
|
def self.call_subscribers(log)
|
|
81
|
-
|
|
106
|
+
subscribers = RactorSupport.other_ractor? ? @ractor_subscribers : @subscribers
|
|
107
|
+
return unless subscribers
|
|
82
108
|
|
|
83
|
-
|
|
109
|
+
subscribers.each do |subscriber|
|
|
84
110
|
subscriber.call(log)
|
|
85
111
|
rescue Exception => e
|
|
86
112
|
processor.logger.error("Exception calling :on_log subscriber", e)
|
|
@@ -7,31 +7,39 @@ end
|
|
|
7
7
|
|
|
8
8
|
# Send Metrics to NewRelic
|
|
9
9
|
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
10
|
+
# Records the metric of each log entry that has one as a New Relic custom metric, through the
|
|
11
|
+
# agent's metric API: with record_metric, in seconds, when the entry has a duration, otherwise with
|
|
12
|
+
# increment_metric, by its metric_amount, or 1. A metric's dimensions are left out, since that API
|
|
13
|
+
# takes none.
|
|
12
14
|
#
|
|
13
15
|
# Example:
|
|
14
16
|
# SemanticLogger.add_appender(metric: :new_relic)
|
|
15
17
|
module SemanticLogger
|
|
16
18
|
module Metric
|
|
17
19
|
class NewRelic < SemanticLogger::Subscriber
|
|
18
|
-
|
|
20
|
+
attr_reader :prefix
|
|
19
21
|
|
|
20
|
-
# Create
|
|
22
|
+
# Create the New Relic metrics subscriber
|
|
21
23
|
#
|
|
22
24
|
# Parameters
|
|
23
|
-
# :
|
|
24
|
-
# Prefix to add to every metric
|
|
25
|
+
# prefix: [String]
|
|
26
|
+
# Prefix to add, with a `/`, to every metric name that does not already start with it and
|
|
27
|
+
# a `/`, before forwarding to NewRelic, which charts custom metrics only under `Custom/`.
|
|
28
|
+
# So 'Customers/signup' is recorded as 'Custom/Customers/signup', and 'Custom/Orders/placed'
|
|
29
|
+
# as it is. Another prefix should start with `Custom/` too, such as 'Custom/MyApp'. A
|
|
30
|
+
# trailing `/` is left out, and an empty prefix adds nothing. Only the default formatter,
|
|
31
|
+
# #call, adds it.
|
|
25
32
|
# Default: 'Custom'
|
|
26
33
|
#
|
|
27
34
|
# level: [:trace | :debug | :info | :warn | :error | :fatal]
|
|
28
|
-
# Override the log level for this
|
|
29
|
-
# Default: :
|
|
35
|
+
# Override the log level for this subscriber.
|
|
36
|
+
# Default: none (:trace), so it records every metric entry that a logger emits; each
|
|
37
|
+
# logger filters by its own level.
|
|
30
38
|
#
|
|
31
39
|
# formatter: [Object|Proc]
|
|
32
|
-
# An instance of a class that implements #call, or a Proc
|
|
33
|
-
# the
|
|
34
|
-
# Default:
|
|
40
|
+
# An instance of a class that implements #call, or a Proc, that returns the name to
|
|
41
|
+
# record the metric under.
|
|
42
|
+
# Default: #call, which adds the prefix to the name of the entry's metric.
|
|
35
43
|
#
|
|
36
44
|
# filter: [Regexp|Proc]
|
|
37
45
|
# RegExp: Only include log messages where the class name matches the supplied.
|
|
@@ -39,16 +47,23 @@ module SemanticLogger
|
|
|
39
47
|
# Proc: Only include log messages where the supplied Proc returns true
|
|
40
48
|
# The Proc must return true or false.
|
|
41
49
|
def initialize(prefix: "Custom", **args, &)
|
|
42
|
-
|
|
50
|
+
self.prefix = prefix
|
|
43
51
|
super(**args, &)
|
|
44
52
|
end
|
|
45
53
|
|
|
54
|
+
# Sets the prefix that #call adds, without a trailing `/`, since #call adds one. An empty
|
|
55
|
+
# prefix is added as an empty String, which every name already starts with.
|
|
56
|
+
def prefix=(prefix)
|
|
57
|
+
@prefix = prefix.to_s.delete_suffix("/")
|
|
58
|
+
@prefix_with_slash = @prefix.empty? ? "" : "#{@prefix}/"
|
|
59
|
+
end
|
|
60
|
+
|
|
46
61
|
# Returns metric name to use.
|
|
47
62
|
def call(log, _logger)
|
|
48
63
|
metric = log.metric
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
metric
|
|
64
|
+
return metric if metric.start_with?(@prefix_with_slash)
|
|
65
|
+
|
|
66
|
+
"#{@prefix_with_slash}#{metric}"
|
|
52
67
|
end
|
|
53
68
|
|
|
54
69
|
def log(log)
|
|
@@ -64,8 +79,8 @@ module SemanticLogger
|
|
|
64
79
|
|
|
65
80
|
# Only forward log entries that contain metrics.
|
|
66
81
|
def should_log?(log)
|
|
67
|
-
#
|
|
68
|
-
log.metric &&
|
|
82
|
+
# Records a metric that has dimensions under its name alone, since the agent's metric API has none.
|
|
83
|
+
log.metric && meets_log_level?(log) && !filtered?(log)
|
|
69
84
|
end
|
|
70
85
|
end
|
|
71
86
|
end
|
|
@@ -19,10 +19,15 @@ module SemanticLogger
|
|
|
19
19
|
# Access Token to use for sending metrics.
|
|
20
20
|
# Obtain the Signalfx token via the Signalfx Web UI under `Organization` -> `Access Tokens`.
|
|
21
21
|
#
|
|
22
|
-
# dimensions: [Array<String>]
|
|
23
|
-
# Dimensions to forward to signalfx when they are present in the named tags of
|
|
24
|
-
#
|
|
25
|
-
#
|
|
22
|
+
# dimensions: [Array<Symbol|String>]
|
|
23
|
+
# Dimensions to forward to signalfx when they are present, and not blank, in the named tags of
|
|
24
|
+
# any log message whose metric has no dimensions of its own. A metric logged with `dimensions:`
|
|
25
|
+
# is sent with those instead, so add the named tags it needs to them.
|
|
26
|
+
# By default `application`, `host`, and, when it is set, `environment` are always included as
|
|
27
|
+
# dimensions in all forwarded metrics.
|
|
28
|
+
# Each value of a dimension makes a new metric time series, so list only named tags whose
|
|
29
|
+
# values are bounded, never a user or request id (see docs/metrics.md).
|
|
30
|
+
# Example: [:region, :plan]
|
|
26
31
|
#
|
|
27
32
|
# filter: [Regexp|Proc]
|
|
28
33
|
# RegExp: Only include log messages where the class name matches the supplied
|
|
@@ -39,8 +44,7 @@ module SemanticLogger
|
|
|
39
44
|
# Default: SemanticLogger.application
|
|
40
45
|
#
|
|
41
46
|
# url: [String]
|
|
42
|
-
# Override the SignalFx service url.
|
|
43
|
-
# For historical data use: https://backfill.signalfx.com/v1/backfill
|
|
47
|
+
# Override the SignalFx service url. The metrics are posted to `v2/datapoint` under its path.
|
|
44
48
|
# Default: https://ingest.signalfx.com
|
|
45
49
|
#
|
|
46
50
|
# Notes:
|
|
@@ -51,16 +55,15 @@ module SemanticLogger
|
|
|
51
55
|
# Using a `count` of a `gauge` in a chart will significantly under-count the number of occurrences.
|
|
52
56
|
#
|
|
53
57
|
# If dimensions are added to the metric, then the metric will be sent as-is and
|
|
54
|
-
# the above logic will _not_ be applied.
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
# logger.info(metric: 'Filters.average', metric_amount: 1.2, dimensions: {user: 'jbloggs'})
|
|
58
|
+
# the above logic will _not_ be applied. A dimension whose value is blank (nil, or text that is
|
|
59
|
+
# empty or only whitespace) is left out, and dimensions that are empty, such as `{}`, or all
|
|
60
|
+
# blank, such as `{state: ""}`, count as none.
|
|
58
61
|
#
|
|
59
62
|
# Example, Counter metric:
|
|
60
|
-
# logger.info(metric: 'Filters.count', dimensions: {
|
|
63
|
+
# logger.info(metric: 'Filters.count', dimensions: {plan: 'premium'})
|
|
61
64
|
#
|
|
62
65
|
# Example, Counter metric with a count other than 1:
|
|
63
|
-
# logger.info(metric: 'Filters.count', metric_amount: 23, dimensions: {
|
|
66
|
+
# logger.info(metric: 'Filters.count', metric_amount: 23, dimensions: {plan: 'premium'})
|
|
64
67
|
#
|
|
65
68
|
# When a duration is supplied and no dimensions are supplied:
|
|
66
69
|
# logger.info(metric: 'Common/User/authorize', duration: 1.4)
|
|
@@ -104,6 +107,12 @@ module SemanticLogger
|
|
|
104
107
|
post(message, full_path)
|
|
105
108
|
end
|
|
106
109
|
|
|
110
|
+
# The #batch above formats the whole batch with #format_batch, so it answers as
|
|
111
|
+
# Appender::Http#batch_refusal does for its own #batch: only while it is the one in use.
|
|
112
|
+
def batch_refusal
|
|
113
|
+
format_batch_refusal if batch_defined_by?(Signalfx)
|
|
114
|
+
end
|
|
115
|
+
|
|
107
116
|
# Only forward log entries that contain metrics.
|
|
108
117
|
def should_log?(log)
|
|
109
118
|
log.metric && meets_log_level?(log) && !filtered?(log)
|
|
@@ -14,30 +14,48 @@ module SemanticLogger
|
|
|
14
14
|
#
|
|
15
15
|
# Parameters:
|
|
16
16
|
# url: [String]
|
|
17
|
-
#
|
|
17
|
+
# Url of the Statsd server to send metrics to. Only udp is supported.
|
|
18
18
|
# Example:
|
|
19
19
|
# udp://localhost:8125
|
|
20
20
|
# Example, send all metrics to a particular namespace:
|
|
21
21
|
# udp://localhost:8125/namespace
|
|
22
22
|
# Default: udp://localhost:8125
|
|
23
23
|
#
|
|
24
|
+
# level, filter: See SemanticLogger::Subscriber, as for the other metric subscribers.
|
|
25
|
+
#
|
|
24
26
|
# Example:
|
|
25
27
|
# SemanticLogger.add_appender(
|
|
26
28
|
# metric: :statsd,
|
|
27
29
|
# url: 'udp://localhost:8125'
|
|
28
30
|
# )
|
|
29
|
-
def initialize(url: "udp://localhost:8125")
|
|
31
|
+
def initialize(url: "udp://localhost:8125", **args, &)
|
|
30
32
|
@url = url
|
|
31
|
-
super()
|
|
33
|
+
super(**args, &)
|
|
34
|
+
reopen
|
|
32
35
|
end
|
|
33
36
|
|
|
37
|
+
# Creates the Statsd client, or reconnects it, such as in a forked process, which closes the socket that the
|
|
38
|
+
# client had, rather than leaving it open. The host is resolved first, so that a failure to resolve it leaves the
|
|
39
|
+
# client connected as it was. Only a url that drops the namespace creates a new client, since a client cannot be
|
|
40
|
+
# given back no namespace, which leaves the old client's socket for garbage collection to close.
|
|
34
41
|
def reopen
|
|
35
42
|
uri = Utils.parse_url(@url)
|
|
36
43
|
raise('Statsd only supports udp. Example: "udp://localhost:8125"') if uri.scheme != "udp"
|
|
37
44
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
45
|
+
# The host without the brackets of an IPv6 address.
|
|
46
|
+
host = uri.hostname
|
|
47
|
+
path = uri.path.chomp("/")
|
|
48
|
+
namespace = path.sub("/", "") unless path == ""
|
|
49
|
+
if @statsd && (namespace || @statsd.namespace.nil?)
|
|
50
|
+
Addrinfo.ip(host)
|
|
51
|
+
@statsd.host = host
|
|
52
|
+
@statsd.port = uri.port
|
|
53
|
+
@statsd.namespace = namespace if namespace
|
|
54
|
+
@statsd.connect
|
|
55
|
+
else
|
|
56
|
+
@statsd = ::Statsd.new(host, uri.port)
|
|
57
|
+
@statsd.namespace = namespace if namespace
|
|
58
|
+
end
|
|
41
59
|
end
|
|
42
60
|
|
|
43
61
|
def log(log)
|
|
@@ -59,8 +77,8 @@ module SemanticLogger
|
|
|
59
77
|
|
|
60
78
|
# Only forward log entries that contain metrics.
|
|
61
79
|
def should_log?(log)
|
|
62
|
-
#
|
|
63
|
-
log.metric &&
|
|
80
|
+
# Records a metric that has dimensions under its name alone, since statsd-ruby has no tags.
|
|
81
|
+
log.metric && meets_log_level?(log) && !filtered?(log)
|
|
64
82
|
end
|
|
65
83
|
|
|
66
84
|
# The url without any credentials. See Subscriber#inspect.
|
|
@@ -1,12 +1,19 @@
|
|
|
1
|
+
# Loaded now rather than autoloaded, since .logger builds one on first use, which can be the first log
|
|
2
|
+
# call of the process, made in a signal trap, where Ruby does not allow a require.
|
|
3
|
+
require "semantic_logger/appender/io"
|
|
4
|
+
|
|
1
5
|
module SemanticLogger
|
|
2
6
|
# Thread that submits and processes log requests
|
|
3
7
|
class Processor < Appender::Async
|
|
4
8
|
# Allow the internal logger to be overridden from its default of $stderr
|
|
5
9
|
# Can be replaced with another Ruby logger or Rails logger, but never to
|
|
6
10
|
# SemanticLogger::Logger itself since it is for reporting problems
|
|
7
|
-
# while trying to log to the various appenders
|
|
8
|
-
|
|
9
|
-
|
|
11
|
+
# while trying to log to the various appenders (raises ArgumentError)
|
|
12
|
+
#
|
|
13
|
+
# A Ruby or Rails logger is wrapped in an appender (see Appender.internal_logger), so .logger
|
|
14
|
+
# returns that appender. The appenders added from then on report to it.
|
|
15
|
+
def self.logger=(logger)
|
|
16
|
+
@logger = Appender.internal_logger(logger)
|
|
10
17
|
end
|
|
11
18
|
|
|
12
19
|
# Internal logger for SemanticLogger
|
|
@@ -24,10 +31,124 @@ module SemanticLogger
|
|
|
24
31
|
attr_reader :appenders
|
|
25
32
|
|
|
26
33
|
def initialize(max_queue_size: 10_000)
|
|
27
|
-
@appenders
|
|
34
|
+
@appenders = Appenders.new(self.class.logger.dup)
|
|
35
|
+
# The appenders whose removal was handed over to the worker thread, and not yet done (see #remove),
|
|
36
|
+
# kept here rather than only on the queue, which a forked child replaces. Takes no lock to change.
|
|
37
|
+
@pending_removals = Concurrent::Array.new
|
|
38
|
+
@pid = ::Process.pid
|
|
28
39
|
super(appender: @appenders, max_queue_size: max_queue_size)
|
|
29
40
|
end
|
|
30
41
|
|
|
42
|
+
# Reopen the appenders, after a fork or within the same process, each on the thread that writes
|
|
43
|
+
# to it, between writes (see Appender::Async#reopen): the worker thread of this processor reopens
|
|
44
|
+
# the appenders (see Appenders#reopen).
|
|
45
|
+
#
|
|
46
|
+
# After a fork, the queue and worker thread of each appender added with async: true or in batches
|
|
47
|
+
# are first replaced on the calling thread, rather than once this processor's worker thread gets
|
|
48
|
+
# to the reopen, so that they are running when this returns, such as for SemanticLogger.stats.
|
|
49
|
+
# That worker thread still asks each of them to reopen its appender, ahead of the entries it
|
|
50
|
+
# hands them. Takes no lock, so that it can be called from a signal trap.
|
|
51
|
+
#
|
|
52
|
+
# In a forked child, the removals handed over to the worker thread that it had not yet done in the
|
|
53
|
+
# parent (see #remove) were on the parent's queue, which the child replaces. So they are handed
|
|
54
|
+
# over again, to the child's worker thread, after the reopen: each such appender, reopened first,
|
|
55
|
+
# then closes only what it built in the child. They count as pending (see #removal_pending?) just
|
|
56
|
+
# before that, so a #remove of one meanwhile returns true while it is still listed, briefly.
|
|
57
|
+
def reopen
|
|
58
|
+
forked = @pid != ::Process.pid
|
|
59
|
+
@pid = ::Process.pid
|
|
60
|
+
appenders.renew_after_fork
|
|
61
|
+
super
|
|
62
|
+
hand_over_pending_removals if forked
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Remove the appender from those that this processor writes to, and close it. Returns the result
|
|
66
|
+
# of closing it, or true when it hands the close over to the worker thread (see below).
|
|
67
|
+
#
|
|
68
|
+
# Before closing an appender that it removed, waits until the log messages already queued are
|
|
69
|
+
# written to the other appenders, and those flushed (see #flush), so that none is still being
|
|
70
|
+
# written to it when it is closed, such as a request on its connection that the worker thread is
|
|
71
|
+
# sending.
|
|
72
|
+
#
|
|
73
|
+
# First, when the appender is one of them, and the appenders are to be reopened (see #reopen) but
|
|
74
|
+
# the worker thread has not yet reopened them, such as in a forked child that has not logged
|
|
75
|
+
# since the fork, waits until it has (see QueueProcessor#await_reopen), so that the appender is
|
|
76
|
+
# reopened before it is removed. Its #close then closes only what it built in this process,
|
|
77
|
+
# rather than what it inherited from the parent, which over TLS would end the parent's connection
|
|
78
|
+
# too, for an appender whose #reopen forgets what the parent built, but whose #close does not. An
|
|
79
|
+
# appender added with async: true or in batches is asked to reopen by that reopen, and its own
|
|
80
|
+
# worker thread reopens it before it runs the close.
|
|
81
|
+
#
|
|
82
|
+
# Called on the worker thread itself, such as by an appender as it writes, it closes the appender
|
|
83
|
+
# at once, without waiting, since nothing else writes to it then.
|
|
84
|
+
#
|
|
85
|
+
# Called where it must not wait for the worker thread (see #current_thread_inside?), such as by an
|
|
86
|
+
# appender added with async: true or in batches as it writes, removing itself or another one, it
|
|
87
|
+
# hands the removal over to the worker thread without waiting (see QueueProcessor#hand_over), and
|
|
88
|
+
# returns true, before the appender is closed. Between writes, never while it writes to the
|
|
89
|
+
# appender, once it has written the log messages queued ahead, the worker thread closes the
|
|
90
|
+
# appender, then removes it (see #close_removed). An appender that removed itself is closed once
|
|
91
|
+
# its current write has returned, after the log messages queued for it.
|
|
92
|
+
#
|
|
93
|
+
# The appender stays in the list until it is closed, so that the thread of an appender that
|
|
94
|
+
# removed itself still counts as one that must not wait while it is being closed, and a close of
|
|
95
|
+
# every appender (see #close) that runs first closes it instead. A removal handed over is never
|
|
96
|
+
# what changes the list while the worker thread goes through it, writing a message or closing
|
|
97
|
+
# every appender, since the worker thread itself takes the appender off the list. Until then,
|
|
98
|
+
# #appenders still lists it.
|
|
99
|
+
#
|
|
100
|
+
# Called on any other thread for an appender whose removal was handed over and is not yet done,
|
|
101
|
+
# it leaves the close to that removal, so that the appender is not closed twice: it waits for the
|
|
102
|
+
# log messages queued ahead to be written (see #flush), which, with the removal queued ahead, takes
|
|
103
|
+
# it off the list, and returns true. Not in a forked child that has not reopened this processor,
|
|
104
|
+
# whose queue still holds none of those removals (see #removal_pending?): it closes the appender.
|
|
105
|
+
#
|
|
106
|
+
# Takes no lock, so that it can be called from a signal trap.
|
|
107
|
+
def remove(appender)
|
|
108
|
+
return hand_over_removal(appender) if current_thread_inside?
|
|
109
|
+
return flush || true if removal_pending?(appender)
|
|
110
|
+
|
|
111
|
+
processor.await_reopen if appenders.include?(appender)
|
|
112
|
+
flush if appenders.delete(appender)
|
|
113
|
+
appender.close
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Write the log messages already queued to the appenders, then flush them (see
|
|
117
|
+
# QueueProcessor#flush).
|
|
118
|
+
#
|
|
119
|
+
# Returns false at once on the worker thread of an appender added with async: true or in batches,
|
|
120
|
+
# such as by that appender as it writes, rather than wait forever: the worker thread of this
|
|
121
|
+
# processor waits for that thread to flush the appender. Reported once to the internal logger.
|
|
122
|
+
def flush
|
|
123
|
+
return super unless current_thread_inside?
|
|
124
|
+
|
|
125
|
+
refuse_on_appender_thread(:flush)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Flush and close the appenders, then remove them (see Appenders#close).
|
|
129
|
+
#
|
|
130
|
+
# Returns false at once on the worker thread of an appender added with async: true or in batches,
|
|
131
|
+
# as #flush does.
|
|
132
|
+
def close
|
|
133
|
+
return super unless current_thread_inside?
|
|
134
|
+
|
|
135
|
+
refuse_on_appender_thread(:close)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Returns [true|false] whether the calling thread must not wait for a thread that uses this
|
|
139
|
+
# processor, since that thread may be waiting for it: the calling thread is the worker thread of
|
|
140
|
+
# an appender added with async: true or in batches, for which the worker thread of this processor
|
|
141
|
+
# waits, to flush or close it, or for room on its full queue. A message that such a thread logs
|
|
142
|
+
# while the queue of this processor is full is dropped, rather than wait for room, only while
|
|
143
|
+
# this processor's worker thread can be waiting for it: while that appender's own queue is full,
|
|
144
|
+
# or a flush or close of it is waiting (see QueueProcessor#holding_up?).
|
|
145
|
+
#
|
|
146
|
+
# Otherwise false: holding no lock while it queues an entry, this processor never has another
|
|
147
|
+
# thread wait for the calling thread to return from a call to it, even on its own worker thread.
|
|
148
|
+
def current_thread_inside?
|
|
149
|
+
appenders.worker_thread?
|
|
150
|
+
end
|
|
151
|
+
|
|
31
152
|
# Start the appender thread
|
|
32
153
|
def start
|
|
33
154
|
return false if active?
|
|
@@ -36,6 +157,11 @@ module SemanticLogger
|
|
|
36
157
|
true
|
|
37
158
|
end
|
|
38
159
|
|
|
160
|
+
# Returns [Integer] the number of log messages waiting on the main pipeline queue.
|
|
161
|
+
def queue_size
|
|
162
|
+
queue.size
|
|
163
|
+
end
|
|
164
|
+
|
|
39
165
|
# Returns [Hash] operational statistics for the logging pipeline.
|
|
40
166
|
#
|
|
41
167
|
# queue_size: [Integer] Number of log messages waiting on the main pipeline queue.
|
|
@@ -47,7 +173,7 @@ module SemanticLogger
|
|
|
47
173
|
# appenders: [Array<Hash>] Per-appender statistics, see Appenders#stats.
|
|
48
174
|
def stats
|
|
49
175
|
{
|
|
50
|
-
queue_size:
|
|
176
|
+
queue_size: queue_size,
|
|
51
177
|
capped: capped?,
|
|
52
178
|
max_queue_size: capped? ? max_queue_size : nil,
|
|
53
179
|
thread_active: active? || false,
|
|
@@ -56,5 +182,68 @@ module SemanticLogger
|
|
|
56
182
|
appenders: appenders.stats
|
|
57
183
|
}
|
|
58
184
|
end
|
|
185
|
+
|
|
186
|
+
private
|
|
187
|
+
|
|
188
|
+
# Hand the removal of the appender over to the worker thread (see #remove), recording it as
|
|
189
|
+
# pending until it is done.
|
|
190
|
+
def hand_over_removal(appender)
|
|
191
|
+
@pending_removals << appender unless @pending_removals.include?(appender)
|
|
192
|
+
processor.hand_over { close_removed(appender) }
|
|
193
|
+
true
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Whether the removal of the appender was handed over to the worker thread (see #remove), and is
|
|
197
|
+
# not yet done. Only in the process that handed it over, or a forked child once #reopen has handed
|
|
198
|
+
# it over again: until then, the queue of a forked child holds the parent's entries, which no
|
|
199
|
+
# worker thread of the child runs.
|
|
200
|
+
def removal_pending?(appender) = @pid == ::Process.pid && @pending_removals.include?(appender)
|
|
201
|
+
|
|
202
|
+
# In a forked child, hand over again the removals pending in the parent (see #reopen), for the
|
|
203
|
+
# appenders still listed.
|
|
204
|
+
def hand_over_pending_removals
|
|
205
|
+
@pending_removals.dup.each do |appender|
|
|
206
|
+
next @pending_removals.delete(appender) unless appenders.include?(appender)
|
|
207
|
+
|
|
208
|
+
processor.hand_over { close_removed(appender) }
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Close the appender, then remove it from the list, on the worker thread, unless it is no longer
|
|
213
|
+
# listed: a close of every appender (see #close), or another removal, already closed it. Its
|
|
214
|
+
# removal is then no longer pending.
|
|
215
|
+
#
|
|
216
|
+
# Reports an error rather than raise it to the worker thread, as Appenders#close does, whatever
|
|
217
|
+
# its class, since one that is not a StandardError would stop the worker thread for good. Except
|
|
218
|
+
# a stack overflow, which the worker thread must not carry on after: on macOS a second one on the
|
|
219
|
+
# same thread crashes Ruby. It is raised, and a new worker thread carries on (see
|
|
220
|
+
# QueueProcessor#run_command).
|
|
221
|
+
def close_removed(appender)
|
|
222
|
+
return @pending_removals.delete(appender) unless appenders.include?(appender)
|
|
223
|
+
|
|
224
|
+
begin
|
|
225
|
+
appender.close
|
|
226
|
+
rescue SystemStackError
|
|
227
|
+
raise
|
|
228
|
+
rescue Exception => e
|
|
229
|
+
appenders.logger.error "Failed to close appender: #{appender.name}", e
|
|
230
|
+
ensure
|
|
231
|
+
appenders.delete(appender)
|
|
232
|
+
@pending_removals.delete(appender)
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Refuse to flush or close on the worker thread of an appender (see #flush), and report it the first
|
|
237
|
+
# time. Returns false.
|
|
238
|
+
def refuse_on_appender_thread(command)
|
|
239
|
+
unless @refused_on_appender_thread
|
|
240
|
+
@refused_on_appender_thread = true
|
|
241
|
+
appenders.logger.warn "SemanticLogger.#{command} was called on the worker thread of an appender " \
|
|
242
|
+
"added with async: true or in batches, such as by that appender as it writes. " \
|
|
243
|
+
"It returns false there, without waiting, since the processor's thread waits " \
|
|
244
|
+
"for that thread. Reported only once."
|
|
245
|
+
end
|
|
246
|
+
false
|
|
247
|
+
end
|
|
59
248
|
end
|
|
60
249
|
end
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
module SemanticLogger
|
|
2
|
+
class QueueProcessor
|
|
3
|
+
# Internal: how long the worker thread of a queue processor waits, after it could not write to
|
|
4
|
+
# the destination of its appender, before it writes again, and when it gives up instead.
|
|
5
|
+
#
|
|
6
|
+
# It knows how many times in a row writing has failed, how many times the worker thread may retry
|
|
7
|
+
# before it gives up (async_max_retries), how many requests, such as a flush or close, are
|
|
8
|
+
# waiting for the worker thread to run them, and whether the process is exiting. Each back-off
|
|
9
|
+
# waits a second longer than the one before, except while the worker thread is waited on, by a
|
|
10
|
+
# request or the exiting process, which must not wait out back-offs: then the worker thread does
|
|
11
|
+
# not wait at all, submitting a request or exiting ends the back-off under way, and once the
|
|
12
|
+
# destination fails, the worker thread gives up on the messages queued ahead of the next request
|
|
13
|
+
# (see #give_up?).
|
|
14
|
+
#
|
|
15
|
+
# Requests are submitted from other threads, and from a signal trap, where Ruby does not allow a
|
|
16
|
+
# Mutex to be locked, so it counts them, and waits, on Queues.
|
|
17
|
+
class Backoff
|
|
18
|
+
# Returns [Integer] the number of times the worker thread retries after a failure, before it
|
|
19
|
+
# gives up, or -1 to retry for ever.
|
|
20
|
+
attr_reader :max_retries
|
|
21
|
+
|
|
22
|
+
# Returns [Integer] the number of failures in a row, which is also how many seconds the next
|
|
23
|
+
# back-off waits.
|
|
24
|
+
attr_reader :retry_count
|
|
25
|
+
|
|
26
|
+
def initialize(max_retries)
|
|
27
|
+
@max_retries = max_retries
|
|
28
|
+
@retry_count = 0
|
|
29
|
+
# One entry for each request waiting.
|
|
30
|
+
@requests = Queue.new
|
|
31
|
+
# Holds an entry once a request has been submitted since the back-off under way began. One is
|
|
32
|
+
# enough to end it, so that submitting many requests without a failure does not add up.
|
|
33
|
+
@interrupts = SizedQueue.new(1)
|
|
34
|
+
@exiting = false
|
|
35
|
+
@gave_up = nil
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Start counting afresh, once writing succeeds.
|
|
39
|
+
def reset
|
|
40
|
+
@retry_count = 0
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Start afresh for a worker thread new to the process, after a fork: it has not failed yet, the
|
|
44
|
+
# requests counted as waiting were on the queue of the parent process, which the child
|
|
45
|
+
# replaces, and the child is not exiting, even when the parent was, such as when another thread
|
|
46
|
+
# forked while the parent flushed at exit.
|
|
47
|
+
def restart
|
|
48
|
+
reset
|
|
49
|
+
@requests.clear
|
|
50
|
+
@exiting = false
|
|
51
|
+
@gave_up = nil
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Count a failure. Returns whether the worker thread retries after it, or false once it has
|
|
55
|
+
# retried max_retries times in a row.
|
|
56
|
+
def retry?
|
|
57
|
+
return false unless max_retries == -1 || retry_count < max_retries
|
|
58
|
+
|
|
59
|
+
@retry_count += 1
|
|
60
|
+
true
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Returns [Integer|String] the number of retries the worker thread makes at most, for a report.
|
|
64
|
+
def limit = max_retries == -1 ? "unlimited" : max_retries
|
|
65
|
+
|
|
66
|
+
# Waits retry_count seconds, unless the worker thread is waited on, or until it is.
|
|
67
|
+
#
|
|
68
|
+
# The interruptions from before are cleared before it checks whether it is waited on, so that a
|
|
69
|
+
# request submitted, or the process exiting, in between either is seen or interrupts the wait.
|
|
70
|
+
def wait
|
|
71
|
+
@interrupts.clear
|
|
72
|
+
@interrupts.pop(timeout: retry_count) unless waited_on?
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Called before a request is added to the queue: it is waiting until #request_done, and ends the
|
|
76
|
+
# back-off under way. Never waits, since an interruption already queued ends it as well.
|
|
77
|
+
def request_submitted
|
|
78
|
+
@requests << true
|
|
79
|
+
@interrupts.push(true, timeout: 0)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Called once for each request submitted, once it has run or will not (see Request#done). The
|
|
83
|
+
# worker thread no longer gives up on the messages queued after it (see #give_up?).
|
|
84
|
+
def request_done
|
|
85
|
+
@requests.pop(timeout: 0)
|
|
86
|
+
@gave_up = nil
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Tell it whether the process is exiting: true before the flush at exit, and false once that
|
|
90
|
+
# flush has returned, since a test framework can run its tests in a later at_exit hook. While
|
|
91
|
+
# the process exits, the worker thread is waited on, as though a request were waiting, and the
|
|
92
|
+
# back-off under way ends. Once it no longer exits, the worker thread gives up no more, unless a
|
|
93
|
+
# request is still waiting (see #gave_up). Called from the exiting thread, which may be in a
|
|
94
|
+
# signal trap, so it takes no lock either.
|
|
95
|
+
def exiting=(exiting)
|
|
96
|
+
@exiting = exiting
|
|
97
|
+
if exiting
|
|
98
|
+
@interrupts.push(true, timeout: 0)
|
|
99
|
+
elsif !waited_on?
|
|
100
|
+
@gave_up = nil
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Whether the worker thread is waited on: a request, such as a flush or close, is waiting for
|
|
105
|
+
# it to run it, or the process is exiting. It then neither waits out back-offs nor writes again
|
|
106
|
+
# what the destination failed.
|
|
107
|
+
def waited_on? = @exiting || request_waiting?
|
|
108
|
+
|
|
109
|
+
# Whether a request, such as a flush or close, is waiting for the worker thread to run it. The
|
|
110
|
+
# thread that submitted it may be the worker thread of the queue processor that writes to this
|
|
111
|
+
# one's appender, which then waits for this worker thread (see QueueProcessor#holding_up?).
|
|
112
|
+
def request_waiting? = !@requests.empty?
|
|
113
|
+
|
|
114
|
+
# Called once the destination failed. While the worker thread is waited on, it gives up on the
|
|
115
|
+
# messages queued ahead of the next request, such as a flush, and discards them without writing
|
|
116
|
+
# them (see #gave_up), since each could take as long to fail, such as the open_timeout of a
|
|
117
|
+
# server that does not answer. While the process exits, it does so as well before the flush at
|
|
118
|
+
# exit reaches the queue. Once a request has run (see #request_done), it writes the next
|
|
119
|
+
# message once more, even while the process exits, and gives up again if that fails. So
|
|
120
|
+
# returns true, having recorded the exception, for the worker thread to carry on straight
|
|
121
|
+
# away, without backing off or counting a retry. Otherwise returns false.
|
|
122
|
+
def give_up?(exception)
|
|
123
|
+
return false unless waited_on?
|
|
124
|
+
|
|
125
|
+
@gave_up = exception
|
|
126
|
+
true
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Returns the exception with which the destination failed while the worker thread was waited
|
|
130
|
+
# on (see #give_up?), while the messages it takes off the queue are to be discarded: until a
|
|
131
|
+
# request has run, or the worker thread is no longer waited on. Otherwise nil.
|
|
132
|
+
def gave_up
|
|
133
|
+
return unless @gave_up
|
|
134
|
+
return @gave_up if waited_on?
|
|
135
|
+
|
|
136
|
+
@gave_up = nil
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|