logstruct 0.1.11 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +146 -1
- data/README.md +76 -68
- data/lib/log_struct/concerns/configuration.rb +139 -91
- data/lib/log_struct/concerns/error_handling.rb +36 -6
- data/lib/log_struct/config_struct/error_handling_modes.rb +3 -1
- data/lib/log_struct/config_struct/filters.rb +126 -22
- data/lib/log_struct/config_struct/integrations.rb +0 -4
- data/lib/log_struct/config_struct/limits.rb +29 -0
- data/lib/log_struct/configuration.rb +2 -0
- data/lib/log_struct/enums/event.rb +3 -1
- data/lib/log_struct/enums/log_field.rb +5 -0
- data/lib/log_struct/formatter.rb +304 -56
- data/lib/log_struct/hash_utils.rb +33 -4
- data/lib/log_struct/integrations/action_mailer/error_handling.rb +78 -20
- data/lib/log_struct/integrations/action_mailer/fallback_error_handlers.rb +81 -0
- data/lib/log_struct/integrations/action_mailer/metadata_collection.rb +0 -20
- data/lib/log_struct/integrations/action_mailer.rb +22 -17
- data/lib/log_struct/integrations/active_job/log_subscriber.rb +177 -25
- data/lib/log_struct/integrations/active_job.rb +2 -7
- data/lib/log_struct/integrations/active_model_serializers.rb +9 -3
- data/lib/log_struct/integrations/active_record.rb +45 -17
- data/lib/log_struct/integrations/active_storage.rb +31 -31
- data/lib/log_struct/integrations/ahoy.rb +19 -15
- data/lib/log_struct/integrations/carrierwave.rb +55 -51
- data/lib/log_struct/integrations/dotenv.rb +22 -32
- data/lib/log_struct/integrations/good_job/logger.rb +30 -24
- data/lib/log_struct/integrations/good_job.rb +111 -40
- data/lib/log_struct/integrations/host_authorization.rb +47 -36
- data/lib/log_struct/integrations/lograge.rb +39 -4
- data/lib/log_struct/integrations/puma.rb +74 -34
- data/lib/log_struct/integrations/rack_error_handler/debug_exceptions_logging.rb +35 -0
- data/lib/log_struct/integrations/rack_error_handler/error_lines.rb +160 -0
- data/lib/log_struct/integrations/rack_error_handler/middleware.rb +14 -169
- data/lib/log_struct/integrations/rack_error_handler/parse_error_logging.rb +36 -0
- data/lib/log_struct/integrations/rack_error_handler.rb +20 -4
- data/lib/log_struct/integrations/rack_middleware.rb +23 -0
- data/lib/log_struct/integrations/rails_log_subscribers.rb +32 -0
- data/lib/log_struct/integrations/request_context/middleware.rb +4 -9
- data/lib/log_struct/integrations/shrine.rb +48 -73
- data/lib/log_struct/integrations/sidekiq/job_context_middleware.rb +32 -0
- data/lib/log_struct/integrations/sidekiq/logger.rb +17 -9
- data/lib/log_struct/integrations/sidekiq.rb +98 -7
- data/lib/log_struct/integrations/sorbet.rb +10 -46
- data/lib/log_struct/integrations.rb +3 -2
- data/lib/log_struct/log/{good_job/enqueue.rb → active_job/discard.rb} +13 -13
- data/lib/log_struct/log/active_job/enqueue.rb +0 -4
- data/lib/log_struct/log/{good_job/schedule.rb → active_job/error.rb} +15 -15
- data/lib/log_struct/log/{good_job/start.rb → active_job/retry.rb} +13 -13
- data/lib/log_struct/log/{good_job/finish.rb → active_job/retry_stopped.rb} +13 -17
- data/lib/log_struct/log/active_job.rb +4 -0
- data/lib/log_struct/log/active_storage/delete.rb +6 -2
- data/lib/log_struct/log/active_storage/download.rb +2 -4
- data/lib/log_struct/log/active_storage/exist.rb +2 -2
- data/lib/log_struct/log/active_storage/metadata.rb +4 -4
- data/lib/log_struct/log/active_storage/upload.rb +2 -10
- data/lib/log_struct/log/active_storage/url.rb +2 -6
- data/lib/log_struct/log/active_storage.rb +2 -3
- data/lib/log_struct/log/error.rb +24 -1
- data/lib/log_struct/log/good_job.rb +0 -4
- data/lib/log_struct/log/puma/error.rb +63 -0
- data/lib/log_struct/log/puma.rb +1 -0
- data/lib/log_struct/log/security/blocked_host.rb +0 -4
- data/lib/log_struct/log/shrine/exist.rb +0 -2
- data/lib/log_struct/log/shrine/metadata.rb +2 -4
- data/lib/log_struct/log/shrine/upload.rb +4 -0
- data/lib/log_struct/log/sidekiq/error.rb +63 -0
- data/lib/log_struct/log/{active_storage/stream.rb → sidekiq/log.rb} +17 -11
- data/lib/log_struct/log/sidekiq.rb +3 -45
- data/lib/log_struct/log.rb +4 -2
- data/lib/log_struct/logged_errors.rb +25 -0
- data/lib/log_struct/monkey_patches/active_support/tagged_logging/formatter.rb +2 -3
- data/lib/log_struct/param_filters.rb +51 -38
- data/lib/log_struct/params_errors.rb +78 -0
- data/lib/log_struct/rails_boot_banner_silencer.rb +7 -2
- data/lib/log_struct/railtie.rb +29 -14
- data/lib/log_struct/semantic_logger/color_formatter.rb +60 -53
- data/lib/log_struct/semantic_logger/concerns/log_methods.rb +65 -57
- data/lib/log_struct/semantic_logger/formatter.rb +79 -57
- data/lib/log_struct/semantic_logger/logger.rb +29 -53
- data/lib/log_struct/semantic_logger/reopen_after_fork.rb +40 -0
- data/lib/log_struct/semantic_logger/setup.rb +115 -112
- data/lib/log_struct/shared/merge_additional_data_fields.rb +14 -1
- data/lib/log_struct/shared/serialize_common.rb +2 -7
- data/lib/log_struct/string_scrubber.rb +229 -20
- data/lib/log_struct/version.rb +1 -1
- data/lib/log_struct.rb +6 -4
- data/logstruct.gemspec +19 -10
- metadata +63 -11
- data/lib/log_struct/builders/active_job.rb +0 -84
- data/lib/log_struct/integrations/good_job/log_subscriber.rb +0 -179
- data/lib/log_struct/integrations/rack.rb +0 -31
|
@@ -16,6 +16,10 @@ module LogStruct
|
|
|
16
16
|
|
|
17
17
|
SHRINE_EVENTS = T.let(%i[upload exists download delete metadata open].freeze, T::Array[Symbol])
|
|
18
18
|
|
|
19
|
+
# The file name in a Content-Disposition (attachment; filename="..."; filename*=UTF-8''...). A quoted name
|
|
20
|
+
# can hold a semicolon, and whatever follows a quote (malformed or not) goes too.
|
|
21
|
+
DISPOSITION_FILENAME_REGEX = /(\bfilename\*?\s*=\s*)(?:"(?:[^"\\]|\\.)*"?)?[^;]*/i
|
|
22
|
+
|
|
19
23
|
# Set up Shrine structured logging
|
|
20
24
|
sig { override.params(config: LogStruct::Configuration).returns(T.nilable(T::Boolean)) }
|
|
21
25
|
def self.setup(config)
|
|
@@ -25,8 +29,8 @@ module LogStruct
|
|
|
25
29
|
|
|
26
30
|
# Create a structured log subscriber for Shrine
|
|
27
31
|
# ActiveSupport::Notifications::Event has name, time, end, transaction_id, payload, and duration
|
|
28
|
-
shrine_log_subscriber =
|
|
29
|
-
payload = event.payload.except(:io, :
|
|
32
|
+
shrine_log_subscriber = lambda do |event|
|
|
33
|
+
payload = event.payload.except(:io, :name).dup
|
|
30
34
|
|
|
31
35
|
# Map event name to Event type
|
|
32
36
|
event_type = case event.name
|
|
@@ -34,9 +38,8 @@ module LogStruct
|
|
|
34
38
|
when :download then Event::Download
|
|
35
39
|
when :open then Event::Download
|
|
36
40
|
when :delete then Event::Delete
|
|
37
|
-
when :metadata then Event::Metadata
|
|
38
41
|
when :exists then Event::Exist
|
|
39
|
-
else Event::
|
|
42
|
+
else Event::Metadata
|
|
40
43
|
end
|
|
41
44
|
|
|
42
45
|
# Create structured log data
|
|
@@ -45,11 +48,15 @@ module LogStruct
|
|
|
45
48
|
|
|
46
49
|
log_data = case event_type
|
|
47
50
|
when Event::Upload
|
|
51
|
+
metadata = payload[:metadata].is_a?(Hash) ? payload[:metadata] : {}
|
|
52
|
+
size = metadata["size"]
|
|
48
53
|
Log::Shrine::Upload.new(
|
|
49
54
|
storage: storage_sym,
|
|
50
55
|
location: payload[:location],
|
|
56
|
+
mime_type: metadata["mime_type"]&.to_s,
|
|
57
|
+
size: size.is_a?(Integer) ? size : nil,
|
|
51
58
|
uploader: payload[:uploader]&.to_s,
|
|
52
|
-
upload_options: payload[:upload_options],
|
|
59
|
+
upload_options: without_file_names(payload[:upload_options]),
|
|
53
60
|
options: payload[:options],
|
|
54
61
|
duration_ms: event.duration.to_f
|
|
55
62
|
)
|
|
@@ -64,95 +71,63 @@ module LogStruct
|
|
|
64
71
|
storage: storage_sym,
|
|
65
72
|
location: payload[:location]
|
|
66
73
|
)
|
|
67
|
-
when Event::Metadata
|
|
68
|
-
metadata_params = {
|
|
69
|
-
storage: storage_sym,
|
|
70
|
-
metadata: payload[:metadata]
|
|
71
|
-
}
|
|
72
|
-
metadata_params[:location] = payload[:location] if payload[:location]
|
|
73
|
-
Log::Shrine::Metadata.new(**metadata_params)
|
|
74
74
|
when Event::Exist
|
|
75
75
|
Log::Shrine::Exist.new(
|
|
76
76
|
storage: storage_sym,
|
|
77
|
-
location: payload[:location]
|
|
78
|
-
exist: payload[:exist]
|
|
77
|
+
location: payload[:location]
|
|
79
78
|
)
|
|
80
79
|
else
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
Log::Shrine::Metadata.new(**unknown_params)
|
|
80
|
+
# metadata, the only other event in SHRINE_EVENTS
|
|
81
|
+
Log::Shrine::Metadata.new(storage: storage_sym, duration_ms: event.duration.to_f)
|
|
84
82
|
end
|
|
85
83
|
|
|
86
84
|
# Log directly through SemanticLogger, NOT through Shrine.logger
|
|
87
85
|
# Shrine.logger is a basic Logger that would just call .to_s on the struct
|
|
88
86
|
::SemanticLogger[::Shrine].info(log_data)
|
|
89
|
-
end)
|
|
90
|
-
|
|
91
|
-
# Check if instrumentation plugin is already loaded
|
|
92
|
-
# If so, we need to replace the existing subscribers, not add duplicates
|
|
93
|
-
if instrumentation_already_configured?
|
|
94
|
-
replace_existing_subscribers(shrine_log_subscriber)
|
|
95
|
-
else
|
|
96
|
-
# First time setup - configure the instrumentation plugin
|
|
97
|
-
::Shrine.plugin :instrumentation,
|
|
98
|
-
events: SHRINE_EVENTS,
|
|
99
|
-
log_subscriber: shrine_log_subscriber
|
|
100
87
|
end
|
|
101
88
|
|
|
102
|
-
|
|
103
|
-
|
|
89
|
+
# Load the instrumentation plugin without Shrine's default log subscriber, which logs
|
|
90
|
+
# each event as a line of text (a no-op if the app has loaded it already). Shrine
|
|
91
|
+
# ignores a subscriber that's already subscribed.
|
|
92
|
+
::Shrine.plugin :instrumentation, log_subscriber: nil
|
|
93
|
+
SHRINE_EVENTS.each do |event_name|
|
|
94
|
+
::Shrine.subscribe(event_name, &shrine_log_subscriber)
|
|
95
|
+
end
|
|
104
96
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
return false unless defined?(::Shrine)
|
|
97
|
+
# The app may load the plugin with the default log subscriber too, before or after this
|
|
98
|
+
::Shrine::Plugins::Instrumentation::LogSubscriber.singleton_class.prepend(SkipLoggedEvents)
|
|
108
99
|
|
|
109
|
-
|
|
110
|
-
|
|
100
|
+
# Other plugins' log subscribers and Shrine's warnings write text to Shrine.logger
|
|
101
|
+
::Shrine.logger = ::SemanticLogger[::Shrine]
|
|
111
102
|
|
|
112
|
-
|
|
113
|
-
|
|
103
|
+
true
|
|
104
|
+
end
|
|
114
105
|
|
|
115
|
-
|
|
116
|
-
|
|
106
|
+
# Upload options with the file names taken out of a Content-Disposition (upload_options:
|
|
107
|
+
# {content_disposition: ...}, which the upload_options plugin often sets): they can name a person
|
|
108
|
+
sig { params(options: T.untyped).returns(T.untyped) }
|
|
109
|
+
def self.without_file_names(options)
|
|
110
|
+
return options unless options.is_a?(Hash)
|
|
117
111
|
|
|
118
|
-
|
|
112
|
+
options.to_h do |key, value|
|
|
113
|
+
if value.is_a?(String) && KeyWords.normalize(key).include?("disposition")
|
|
114
|
+
[key, value.gsub(DISPOSITION_FILENAME_REGEX, '\1[FILENAME]')]
|
|
115
|
+
else
|
|
116
|
+
[key, value]
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
119
|
end
|
|
120
120
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
subscribers = instrumentation_opts[:subscribers]
|
|
121
|
+
# Prepended to Shrine's default instrumentation log subscriber, so it doesn't log the
|
|
122
|
+
# events that LogStruct logs
|
|
123
|
+
module SkipLoggedEvents
|
|
124
|
+
extend T::Sig
|
|
126
125
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
subscribers[event_name] = [] if subscribers[event_name]
|
|
131
|
-
|
|
132
|
-
# Add our subscriber
|
|
133
|
-
subscribers[event_name] ||= []
|
|
134
|
-
subscribers[event_name] << new_subscriber
|
|
135
|
-
|
|
136
|
-
# Also re-subscribe via ActiveSupport::Notifications
|
|
137
|
-
# Shrine uses "shrine.#{event_name}" as the notification name
|
|
138
|
-
notification_name = "shrine.#{event_name}"
|
|
139
|
-
|
|
140
|
-
# Unsubscribe existing listeners for this event
|
|
141
|
-
# ActiveSupport::Notifications stores subscriptions, we need to find and remove them
|
|
142
|
-
notifier = ::ActiveSupport::Notifications.notifier
|
|
143
|
-
if notifier.respond_to?(:listeners_for)
|
|
144
|
-
# Rails 7.0+ uses listeners_for
|
|
145
|
-
listeners = notifier.listeners_for(notification_name)
|
|
146
|
-
listeners.each do |listener|
|
|
147
|
-
::ActiveSupport::Notifications.unsubscribe(listener)
|
|
148
|
-
end
|
|
149
|
-
end
|
|
126
|
+
sig { params(event: T.untyped).returns(T.untyped) }
|
|
127
|
+
def call(event)
|
|
128
|
+
return if SHRINE_EVENTS.include?(event.name)
|
|
150
129
|
|
|
151
|
-
|
|
152
|
-
::ActiveSupport::Notifications.subscribe(notification_name) do |*args|
|
|
153
|
-
event = ::ActiveSupport::Notifications::Event.new(*args)
|
|
154
|
-
new_subscriber.call(event)
|
|
155
|
-
end
|
|
130
|
+
super
|
|
156
131
|
end
|
|
157
132
|
end
|
|
158
133
|
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module LogStruct
|
|
5
|
+
module Integrations
|
|
6
|
+
module Sidekiq
|
|
7
|
+
# Sidekiq server middleware that adds the running job's ID and class to every line
|
|
8
|
+
# logged while it runs, in the same `ctx` field as Sidekiq's own log lines. A query
|
|
9
|
+
# on ctx.jid then returns every line for one job run.
|
|
10
|
+
class JobContextMiddleware
|
|
11
|
+
extend T::Sig
|
|
12
|
+
|
|
13
|
+
sig do
|
|
14
|
+
params(
|
|
15
|
+
job_instance: T.untyped,
|
|
16
|
+
job_hash: T::Hash[String, T.untyped],
|
|
17
|
+
queue: String,
|
|
18
|
+
block: T.proc.returns(T.untyped)
|
|
19
|
+
).returns(T.untyped)
|
|
20
|
+
end
|
|
21
|
+
def call(job_instance, job_hash, queue, &block)
|
|
22
|
+
context = {
|
|
23
|
+
jid: job_hash["jid"],
|
|
24
|
+
# Same precedence as Sidekiq's own context, so ActiveJob jobs show their class
|
|
25
|
+
class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"]
|
|
26
|
+
}
|
|
27
|
+
::SemanticLogger.named_tagged(sidekiq_context: context, &block)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -16,32 +16,40 @@ module LogStruct
|
|
|
16
16
|
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
# Custom Logger for Sidekiq that creates LogStruct::Log::Sidekiq entries
|
|
19
|
+
# Custom Logger for Sidekiq that creates LogStruct::Log::Sidekiq::Log entries
|
|
20
20
|
class Logger < LogStruct::SemanticLogger::Logger
|
|
21
21
|
extend T::Sig
|
|
22
22
|
|
|
23
23
|
# Override log methods to create Sidekiq log structs
|
|
24
24
|
%i[debug info warn error fatal].each do |level|
|
|
25
25
|
define_method(level) do |message = nil, payload = nil, &block|
|
|
26
|
-
#
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
message.to_s
|
|
26
|
+
# Hashes, LogStruct log entries and exceptions are logged as they are, so their keys are
|
|
27
|
+
# filtered (Sidekiq logs some settings as a Hash)
|
|
28
|
+
unless message.nil? || message.is_a?(String)
|
|
29
|
+
next super(message, payload, &block)
|
|
31
30
|
end
|
|
31
|
+
# A message with an exception is an error line with both
|
|
32
|
+
next super(message, payload, &block) if payload.is_a?(Exception)
|
|
33
|
+
|
|
34
|
+
msg = message || (block ? block.call.to_s : "")
|
|
35
|
+
# Lines logged outside a job have no context
|
|
36
|
+
context = ::Sidekiq::Context.current
|
|
32
37
|
|
|
33
38
|
# Create a Sidekiq log struct with the message
|
|
34
|
-
log_struct = Log::Sidekiq.new(
|
|
39
|
+
log_struct = Log::Sidekiq::Log.new(
|
|
35
40
|
level: LogStruct::Level.from_severity(level.to_s.upcase),
|
|
36
41
|
event: Event::Log,
|
|
37
42
|
message: msg,
|
|
38
43
|
process_id: ::Process.pid,
|
|
39
44
|
thread_id: Sidekiq.tid,
|
|
40
|
-
|
|
45
|
+
# Log a copy: SemanticLogger may still be serializing the struct on its appender
|
|
46
|
+
# thread when Sidekiq adds :elapsed to the live context after the job finishes.
|
|
47
|
+
context: context.blank? ? nil : context.dup,
|
|
48
|
+
additional_data: payload.is_a?(Hash) ? payload : nil
|
|
41
49
|
)
|
|
42
50
|
|
|
43
51
|
# Pass the struct to SemanticLogger
|
|
44
|
-
super(log_struct,
|
|
52
|
+
super(log_struct, nil, &nil)
|
|
45
53
|
end
|
|
46
54
|
end
|
|
47
55
|
end
|
|
@@ -6,7 +6,10 @@ begin
|
|
|
6
6
|
rescue LoadError
|
|
7
7
|
# Sidekiq gem is not available, integration will be skipped
|
|
8
8
|
end
|
|
9
|
-
|
|
9
|
+
if defined?(::Sidekiq)
|
|
10
|
+
require_relative "sidekiq/logger"
|
|
11
|
+
require_relative "sidekiq/job_context_middleware"
|
|
12
|
+
end
|
|
10
13
|
|
|
11
14
|
module LogStruct
|
|
12
15
|
module Integrations
|
|
@@ -15,6 +18,8 @@ module LogStruct
|
|
|
15
18
|
extend T::Sig
|
|
16
19
|
extend IntegrationInterface
|
|
17
20
|
|
|
21
|
+
@original_logger = T.let(nil, T.untyped)
|
|
22
|
+
|
|
18
23
|
# Set up Sidekiq structured logging
|
|
19
24
|
sig { override.params(config: LogStruct::Configuration).returns(T.nilable(T::Boolean)) }
|
|
20
25
|
def self.setup(config)
|
|
@@ -22,18 +27,104 @@ module LogStruct
|
|
|
22
27
|
return nil unless config.enabled
|
|
23
28
|
return nil unless config.integrations.enable_sidekiq
|
|
24
29
|
|
|
25
|
-
|
|
30
|
+
configure_logger
|
|
31
|
+
|
|
26
32
|
::Sidekiq.configure_server do |sidekiq_config|
|
|
27
|
-
|
|
28
|
-
|
|
33
|
+
# First in the chain, so lines logged by other middleware include the job too
|
|
34
|
+
sidekiq_config.server_middleware do |chain|
|
|
35
|
+
chain.prepend LogStruct::Integrations::Sidekiq::JobContextMiddleware
|
|
36
|
+
end
|
|
29
37
|
|
|
30
|
-
|
|
31
|
-
::Sidekiq.configure_client do |sidekiq_config|
|
|
32
|
-
sidekiq_config.logger = LogStruct::Integrations::Sidekiq::Logger.new("Sidekiq-Client")
|
|
38
|
+
replace_default_error_handler(sidekiq_config)
|
|
33
39
|
end
|
|
34
40
|
|
|
35
41
|
true
|
|
36
42
|
end
|
|
43
|
+
|
|
44
|
+
# Use LogStruct's logger for Sidekiq's own lines. The Railtie calls this as soon as it replaces
|
|
45
|
+
# Rails.logger, before config/initializers run, because an initializer that configures Sidekiq
|
|
46
|
+
# (connecting to Redis, setting up a fetcher) logs with Sidekiq's logger straight away.
|
|
47
|
+
sig { void }
|
|
48
|
+
def self.configure_logger
|
|
49
|
+
return unless defined?(::Sidekiq)
|
|
50
|
+
|
|
51
|
+
current = sidekiq_logger_config.logger
|
|
52
|
+
return if current.is_a?(LogStruct::Integrations::Sidekiq::Logger)
|
|
53
|
+
|
|
54
|
+
@original_logger = current
|
|
55
|
+
name = ::Sidekiq.server? ? "Sidekiq-Server" : "Sidekiq-Client"
|
|
56
|
+
sidekiq_logger_config.logger = LogStruct::Integrations::Sidekiq::Logger.new(name)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Put Sidekiq's own logger back, when an initializer turned LogStruct or this integration off
|
|
60
|
+
# after configure_logger ran
|
|
61
|
+
sig { void }
|
|
62
|
+
def self.restore_logger
|
|
63
|
+
return unless defined?(::Sidekiq)
|
|
64
|
+
return if @original_logger.nil?
|
|
65
|
+
|
|
66
|
+
sidekiq_logger_config.logger = @original_logger
|
|
67
|
+
@original_logger = nil
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Sidekiq 7 keeps the logger on its default configuration, Sidekiq 6 on the module
|
|
71
|
+
sig { returns(T.untyped) }
|
|
72
|
+
def self.sidekiq_logger_config
|
|
73
|
+
::Sidekiq.respond_to?(:default_configuration) ? ::Sidekiq.default_configuration : ::Sidekiq
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Sidekiq's default error handler logs three lines for each failed job: the job (with its
|
|
77
|
+
# arguments) as a JSON string, the error, and the backtrace. Log one structured line instead,
|
|
78
|
+
# without the arguments, which LogStruct can't filter inside a JSON string.
|
|
79
|
+
sig { params(sidekiq_config: T.untyped).void }
|
|
80
|
+
def self.replace_default_error_handler(sidekiq_config)
|
|
81
|
+
return unless defined?(::Sidekiq::Config::ERROR_HANDLER)
|
|
82
|
+
|
|
83
|
+
handlers = sidekiq_config.error_handlers
|
|
84
|
+
return unless handlers.delete(::Sidekiq::Config::ERROR_HANDLER)
|
|
85
|
+
|
|
86
|
+
handlers.unshift(JOB_ERROR_HANDLER)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# The line has the job's ID and class in ctx, like the lines logged while the job ran (Sidekiq
|
|
90
|
+
# calls its error handlers after the job's middleware has finished).
|
|
91
|
+
sig { params(error: Exception, context: T::Hash[T.untyped, T.untyped]).void }
|
|
92
|
+
def self.log_job_error(error, context)
|
|
93
|
+
# LogStruct's ActiveJob integration logs the errors it sees an ActiveJob job raise. An
|
|
94
|
+
# error raised before ActiveJob ran the job (a job class a deploy removed, say) is logged
|
|
95
|
+
# here.
|
|
96
|
+
return if LogStruct::LoggedErrors.logged?(error)
|
|
97
|
+
|
|
98
|
+
job = context[:job]
|
|
99
|
+
job = nil unless job.is_a?(Hash)
|
|
100
|
+
error_context = context[:context]
|
|
101
|
+
|
|
102
|
+
line = Log::Sidekiq::Error.new(
|
|
103
|
+
error_class: error.class,
|
|
104
|
+
message: error.message,
|
|
105
|
+
backtrace: error.backtrace,
|
|
106
|
+
error_context: error_context&.to_s,
|
|
107
|
+
queue_name: job&.dig("queue")&.to_s,
|
|
108
|
+
retry_count: job&.dig("retry_count")
|
|
109
|
+
)
|
|
110
|
+
if job.nil?
|
|
111
|
+
::Rails.logger.warn(line)
|
|
112
|
+
return
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
job_context = {
|
|
116
|
+
jid: job["jid"],
|
|
117
|
+
class: job["display_class"] || job["wrapped"] || job["class"]
|
|
118
|
+
}.compact
|
|
119
|
+
::SemanticLogger.named_tagged(sidekiq_context: job_context) { ::Rails.logger.warn(line) }
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
JOB_ERROR_HANDLER = T.let(
|
|
123
|
+
->(error, context, _config = nil) { log_job_error(error, context) },
|
|
124
|
+
T.proc.params(error: Exception, context: T::Hash[T.untyped, T.untyped], config: T.untyped).void
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
private_class_method :sidekiq_logger_config, :replace_default_error_handler, :log_job_error
|
|
37
128
|
end
|
|
38
129
|
end
|
|
39
130
|
end
|
|
@@ -17,12 +17,10 @@ module LogStruct
|
|
|
17
17
|
def self.setup(config)
|
|
18
18
|
return nil unless config.integrations.enable_sorbet_error_handlers
|
|
19
19
|
|
|
20
|
-
clear_sig_error_handler!
|
|
21
|
-
install_error_handler!
|
|
22
|
-
|
|
23
20
|
# Install inline type error handler
|
|
24
21
|
# Called when T.let, T.cast, T.must, etc. fail
|
|
25
22
|
T::Configuration.inline_type_error_handler = lambda do |error, _opts|
|
|
23
|
+
add_backtrace(error)
|
|
26
24
|
LogStruct.handle_exception(error, source: LogStruct::Source::TypeChecking)
|
|
27
25
|
end
|
|
28
26
|
|
|
@@ -30,67 +28,33 @@ module LogStruct
|
|
|
30
28
|
# Called when method signature validation fails
|
|
31
29
|
T::Configuration.call_validation_error_handler = lambda do |_signature, opts|
|
|
32
30
|
error = TypeError.new(opts[:pretty_message])
|
|
31
|
+
add_backtrace(error)
|
|
33
32
|
LogStruct.handle_exception(error, source: LogStruct::Source::TypeChecking)
|
|
34
33
|
end
|
|
35
34
|
|
|
36
35
|
# Install sig builder error handler
|
|
37
36
|
# Called when there's a problem with a signature definition
|
|
38
37
|
T::Configuration.sig_builder_error_handler = lambda do |error, _location|
|
|
38
|
+
add_backtrace(error)
|
|
39
39
|
LogStruct.handle_exception(error, source: LogStruct::Source::TypeChecking)
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
# Install sig validation error handler
|
|
43
43
|
# Called when there's a problem with a signature validation
|
|
44
44
|
T::Configuration.sig_validation_error_handler = lambda do |error, _opts|
|
|
45
|
+
add_backtrace(error)
|
|
45
46
|
LogStruct.handle_exception(error, source: LogStruct::Source::TypeChecking)
|
|
46
47
|
end
|
|
47
48
|
|
|
48
49
|
true
|
|
49
50
|
end
|
|
50
51
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
sig { void }
|
|
59
|
-
def install_error_handler!
|
|
60
|
-
return if installed?
|
|
61
|
-
|
|
62
|
-
T::Configuration.sig_builder_error_handler = lambda do |error, source|
|
|
63
|
-
LogStruct.handle_exception(error, source: source, context: nil)
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
@installed = true
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
sig do
|
|
70
|
-
returns(
|
|
71
|
-
T.nilable(
|
|
72
|
-
T.proc.params(error: StandardError, location: Thread::Backtrace::Location).void
|
|
73
|
-
)
|
|
74
|
-
)
|
|
75
|
-
end
|
|
76
|
-
def clear_sig_error_handler!
|
|
77
|
-
previous_handler = T.cast(
|
|
78
|
-
T::Configuration.instance_variable_get(:@sig_builder_error_handler),
|
|
79
|
-
T.nilable(
|
|
80
|
-
T.proc.params(error: StandardError, location: Thread::Backtrace::Location).void
|
|
81
|
-
)
|
|
82
|
-
)
|
|
83
|
-
T::Configuration.sig_builder_error_handler = nil
|
|
84
|
-
|
|
85
|
-
@installed = false
|
|
86
|
-
|
|
87
|
-
previous_handler
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
sig { returns(T::Boolean) }
|
|
91
|
-
def installed?
|
|
92
|
-
@installed
|
|
93
|
-
end
|
|
52
|
+
# Starts the error's backtrace at the code that failed, below sorbet-runtime's frames (and this file's).
|
|
53
|
+
# sorbet-runtime passes some errors without raising them, which have no backtrace at all.
|
|
54
|
+
sig { params(error: Exception).void }
|
|
55
|
+
def self.add_backtrace(error)
|
|
56
|
+
frames = error.backtrace || caller
|
|
57
|
+
error.set_backtrace(frames.drop_while { |frame| frame.include?(__FILE__) || frame.include?("sorbet-runtime") })
|
|
94
58
|
end
|
|
95
59
|
end
|
|
96
60
|
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "integrations/integration_interface"
|
|
5
|
+
require_relative "integrations/rails_log_subscribers"
|
|
5
6
|
require_relative "integrations/request_context"
|
|
6
7
|
require_relative "integrations/active_job"
|
|
7
8
|
require_relative "integrations/active_record"
|
|
@@ -33,8 +34,8 @@ module LogStruct
|
|
|
33
34
|
LogStruct::Integrations::Dotenv.intercept_logger_setter!
|
|
34
35
|
end
|
|
35
36
|
|
|
36
|
-
# Decide which set of boot logs to emit
|
|
37
|
-
railtie.initializer "logstruct.resolve_boot_logs", after:
|
|
37
|
+
# Decide which set of boot logs to emit once `enabled` is final
|
|
38
|
+
railtie.initializer "logstruct.resolve_boot_logs", after: "logstruct.setup" do
|
|
38
39
|
LogStruct::Integrations::Dotenv.resolve_boot_logs!
|
|
39
40
|
end
|
|
40
41
|
end
|
|
@@ -19,26 +19,26 @@ require_relative "../../enums/log_field"
|
|
|
19
19
|
|
|
20
20
|
module LogStruct
|
|
21
21
|
module Log
|
|
22
|
-
class
|
|
23
|
-
class
|
|
22
|
+
class ActiveJob
|
|
23
|
+
class Discard < T::Struct
|
|
24
24
|
extend T::Sig
|
|
25
25
|
|
|
26
26
|
# Shared/common fields
|
|
27
27
|
const :source, Source::Job, default: Source::Job
|
|
28
|
-
const :event, Event, default: Event::
|
|
28
|
+
const :event, Event, default: Event::Discard
|
|
29
29
|
const :timestamp, Time, factory: -> { Time.now }
|
|
30
30
|
const :level, Level, default: Level::Info
|
|
31
31
|
|
|
32
|
-
const :job_id,
|
|
33
|
-
const :job_class,
|
|
32
|
+
const :job_id, String
|
|
33
|
+
const :job_class, String
|
|
34
34
|
const :queue_name, T.nilable(Symbol), default: nil
|
|
35
35
|
const :arguments, T.nilable(T::Array[T.untyped]), default: nil
|
|
36
36
|
const :executions, T.nilable(Integer), default: nil
|
|
37
|
+
const :provider_job_id, T.nilable(String), default: nil
|
|
37
38
|
|
|
38
39
|
# Event-specific fields
|
|
39
|
-
const :
|
|
40
|
-
const :
|
|
41
|
-
const :enqueue_caller, T.nilable(String), default: nil
|
|
40
|
+
const :error_class, T.class_of(Exception)
|
|
41
|
+
const :message, String
|
|
42
42
|
|
|
43
43
|
# Serialize shared fields
|
|
44
44
|
include LogStruct::Log::Interfaces::CommonFields
|
|
@@ -47,14 +47,14 @@ module LogStruct
|
|
|
47
47
|
sig { returns(T::Hash[LogStruct::LogField, T.untyped]) }
|
|
48
48
|
def to_h
|
|
49
49
|
h = T.let({}, T::Hash[LogStruct::LogField, T.untyped])
|
|
50
|
-
h[LogField::JobId] = job_id
|
|
51
|
-
h[LogField::JobClass] = job_class
|
|
50
|
+
h[LogField::JobId] = job_id
|
|
51
|
+
h[LogField::JobClass] = job_class
|
|
52
52
|
h[LogField::QueueName] = queue_name unless queue_name.nil?
|
|
53
53
|
h[LogField::Arguments] = arguments unless arguments.nil?
|
|
54
54
|
h[LogField::Executions] = executions unless executions.nil?
|
|
55
|
-
h[LogField::
|
|
56
|
-
h[LogField::
|
|
57
|
-
h[LogField::
|
|
55
|
+
h[LogField::ProviderJobId] = provider_job_id unless provider_job_id.nil?
|
|
56
|
+
h[LogField::ErrorClass] = error_class
|
|
57
|
+
h[LogField::Message] = message
|
|
58
58
|
h
|
|
59
59
|
end
|
|
60
60
|
end
|
|
@@ -36,9 +36,6 @@ module LogStruct
|
|
|
36
36
|
const :executions, T.nilable(Integer), default: nil
|
|
37
37
|
const :provider_job_id, T.nilable(String), default: nil
|
|
38
38
|
|
|
39
|
-
# Event-specific fields
|
|
40
|
-
const :retries, T.nilable(Integer), default: nil
|
|
41
|
-
|
|
42
39
|
# Serialize shared fields
|
|
43
40
|
include LogStruct::Log::Interfaces::CommonFields
|
|
44
41
|
include LogStruct::Log::Shared::SerializeCommon
|
|
@@ -52,7 +49,6 @@ module LogStruct
|
|
|
52
49
|
h[LogField::Arguments] = arguments unless arguments.nil?
|
|
53
50
|
h[LogField::Executions] = executions unless executions.nil?
|
|
54
51
|
h[LogField::ProviderJobId] = provider_job_id unless provider_job_id.nil?
|
|
55
|
-
h[LogField::Retries] = retries unless retries.nil?
|
|
56
52
|
h
|
|
57
53
|
end
|
|
58
54
|
end
|
|
@@ -19,27 +19,27 @@ require_relative "../../enums/log_field"
|
|
|
19
19
|
|
|
20
20
|
module LogStruct
|
|
21
21
|
module Log
|
|
22
|
-
class
|
|
23
|
-
class
|
|
22
|
+
class ActiveJob
|
|
23
|
+
class Error < T::Struct
|
|
24
24
|
extend T::Sig
|
|
25
25
|
|
|
26
26
|
# Shared/common fields
|
|
27
27
|
const :source, Source::Job, default: Source::Job
|
|
28
|
-
const :event, Event, default: Event::
|
|
28
|
+
const :event, Event, default: Event::Error
|
|
29
29
|
const :timestamp, Time, factory: -> { Time.now }
|
|
30
30
|
const :level, Level, default: Level::Info
|
|
31
31
|
|
|
32
|
-
const :job_id,
|
|
33
|
-
const :job_class,
|
|
32
|
+
const :job_id, String
|
|
33
|
+
const :job_class, String
|
|
34
34
|
const :queue_name, T.nilable(Symbol), default: nil
|
|
35
35
|
const :arguments, T.nilable(T::Array[T.untyped]), default: nil
|
|
36
36
|
const :executions, T.nilable(Integer), default: nil
|
|
37
|
+
const :provider_job_id, T.nilable(String), default: nil
|
|
37
38
|
|
|
38
39
|
# Event-specific fields
|
|
39
|
-
const :
|
|
40
|
-
const :
|
|
41
|
-
const :
|
|
42
|
-
const :cron_key, T.nilable(String), default: nil
|
|
40
|
+
const :error_class, T.class_of(Exception)
|
|
41
|
+
const :message, String
|
|
42
|
+
const :backtrace, T.nilable(T::Array[String]), default: nil
|
|
43
43
|
|
|
44
44
|
# Serialize shared fields
|
|
45
45
|
include LogStruct::Log::Interfaces::CommonFields
|
|
@@ -48,15 +48,15 @@ module LogStruct
|
|
|
48
48
|
sig { returns(T::Hash[LogStruct::LogField, T.untyped]) }
|
|
49
49
|
def to_h
|
|
50
50
|
h = T.let({}, T::Hash[LogStruct::LogField, T.untyped])
|
|
51
|
-
h[LogField::JobId] = job_id
|
|
52
|
-
h[LogField::JobClass] = job_class
|
|
51
|
+
h[LogField::JobId] = job_id
|
|
52
|
+
h[LogField::JobClass] = job_class
|
|
53
53
|
h[LogField::QueueName] = queue_name unless queue_name.nil?
|
|
54
54
|
h[LogField::Arguments] = arguments unless arguments.nil?
|
|
55
55
|
h[LogField::Executions] = executions unless executions.nil?
|
|
56
|
-
h[LogField::
|
|
57
|
-
h[LogField::
|
|
58
|
-
h[LogField::
|
|
59
|
-
h[LogField::
|
|
56
|
+
h[LogField::ProviderJobId] = provider_job_id unless provider_job_id.nil?
|
|
57
|
+
h[LogField::ErrorClass] = error_class
|
|
58
|
+
h[LogField::Message] = message
|
|
59
|
+
h[LogField::Backtrace] = backtrace unless backtrace.nil?
|
|
60
60
|
h
|
|
61
61
|
end
|
|
62
62
|
end
|