logstruct 0.1.11 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +146 -1
- data/README.md +76 -68
- data/lib/log_struct/concerns/configuration.rb +139 -91
- data/lib/log_struct/concerns/error_handling.rb +36 -6
- data/lib/log_struct/config_struct/error_handling_modes.rb +3 -1
- data/lib/log_struct/config_struct/filters.rb +126 -22
- data/lib/log_struct/config_struct/integrations.rb +0 -4
- data/lib/log_struct/config_struct/limits.rb +29 -0
- data/lib/log_struct/configuration.rb +2 -0
- data/lib/log_struct/enums/event.rb +3 -1
- data/lib/log_struct/enums/log_field.rb +5 -0
- data/lib/log_struct/formatter.rb +304 -56
- data/lib/log_struct/hash_utils.rb +33 -4
- data/lib/log_struct/integrations/action_mailer/error_handling.rb +78 -20
- data/lib/log_struct/integrations/action_mailer/fallback_error_handlers.rb +81 -0
- data/lib/log_struct/integrations/action_mailer/metadata_collection.rb +0 -20
- data/lib/log_struct/integrations/action_mailer.rb +22 -17
- data/lib/log_struct/integrations/active_job/log_subscriber.rb +177 -25
- data/lib/log_struct/integrations/active_job.rb +2 -7
- data/lib/log_struct/integrations/active_model_serializers.rb +9 -3
- data/lib/log_struct/integrations/active_record.rb +45 -17
- data/lib/log_struct/integrations/active_storage.rb +31 -31
- data/lib/log_struct/integrations/ahoy.rb +19 -15
- data/lib/log_struct/integrations/carrierwave.rb +55 -51
- data/lib/log_struct/integrations/dotenv.rb +22 -32
- data/lib/log_struct/integrations/good_job/logger.rb +30 -24
- data/lib/log_struct/integrations/good_job.rb +111 -40
- data/lib/log_struct/integrations/host_authorization.rb +47 -36
- data/lib/log_struct/integrations/lograge.rb +39 -4
- data/lib/log_struct/integrations/puma.rb +74 -34
- data/lib/log_struct/integrations/rack_error_handler/debug_exceptions_logging.rb +35 -0
- data/lib/log_struct/integrations/rack_error_handler/error_lines.rb +160 -0
- data/lib/log_struct/integrations/rack_error_handler/middleware.rb +14 -169
- data/lib/log_struct/integrations/rack_error_handler/parse_error_logging.rb +36 -0
- data/lib/log_struct/integrations/rack_error_handler.rb +20 -4
- data/lib/log_struct/integrations/rack_middleware.rb +23 -0
- data/lib/log_struct/integrations/rails_log_subscribers.rb +32 -0
- data/lib/log_struct/integrations/request_context/middleware.rb +4 -9
- data/lib/log_struct/integrations/shrine.rb +48 -73
- data/lib/log_struct/integrations/sidekiq/job_context_middleware.rb +32 -0
- data/lib/log_struct/integrations/sidekiq/logger.rb +17 -9
- data/lib/log_struct/integrations/sidekiq.rb +98 -7
- data/lib/log_struct/integrations/sorbet.rb +10 -46
- data/lib/log_struct/integrations.rb +3 -2
- data/lib/log_struct/log/{good_job/enqueue.rb → active_job/discard.rb} +13 -13
- data/lib/log_struct/log/active_job/enqueue.rb +0 -4
- data/lib/log_struct/log/{good_job/schedule.rb → active_job/error.rb} +15 -15
- data/lib/log_struct/log/{good_job/start.rb → active_job/retry.rb} +13 -13
- data/lib/log_struct/log/{good_job/finish.rb → active_job/retry_stopped.rb} +13 -17
- data/lib/log_struct/log/active_job.rb +4 -0
- data/lib/log_struct/log/active_storage/delete.rb +6 -2
- data/lib/log_struct/log/active_storage/download.rb +2 -4
- data/lib/log_struct/log/active_storage/exist.rb +2 -2
- data/lib/log_struct/log/active_storage/metadata.rb +4 -4
- data/lib/log_struct/log/active_storage/upload.rb +2 -10
- data/lib/log_struct/log/active_storage/url.rb +2 -6
- data/lib/log_struct/log/active_storage.rb +2 -3
- data/lib/log_struct/log/error.rb +24 -1
- data/lib/log_struct/log/good_job.rb +0 -4
- data/lib/log_struct/log/puma/error.rb +63 -0
- data/lib/log_struct/log/puma.rb +1 -0
- data/lib/log_struct/log/security/blocked_host.rb +0 -4
- data/lib/log_struct/log/shrine/exist.rb +0 -2
- data/lib/log_struct/log/shrine/metadata.rb +2 -4
- data/lib/log_struct/log/shrine/upload.rb +4 -0
- data/lib/log_struct/log/sidekiq/error.rb +63 -0
- data/lib/log_struct/log/{active_storage/stream.rb → sidekiq/log.rb} +17 -11
- data/lib/log_struct/log/sidekiq.rb +3 -45
- data/lib/log_struct/log.rb +4 -2
- data/lib/log_struct/logged_errors.rb +25 -0
- data/lib/log_struct/monkey_patches/active_support/tagged_logging/formatter.rb +2 -3
- data/lib/log_struct/param_filters.rb +51 -38
- data/lib/log_struct/params_errors.rb +78 -0
- data/lib/log_struct/rails_boot_banner_silencer.rb +7 -2
- data/lib/log_struct/railtie.rb +29 -14
- data/lib/log_struct/semantic_logger/color_formatter.rb +60 -53
- data/lib/log_struct/semantic_logger/concerns/log_methods.rb +65 -57
- data/lib/log_struct/semantic_logger/formatter.rb +79 -57
- data/lib/log_struct/semantic_logger/logger.rb +29 -53
- data/lib/log_struct/semantic_logger/reopen_after_fork.rb +40 -0
- data/lib/log_struct/semantic_logger/setup.rb +115 -112
- data/lib/log_struct/shared/merge_additional_data_fields.rb +14 -1
- data/lib/log_struct/shared/serialize_common.rb +2 -7
- data/lib/log_struct/string_scrubber.rb +229 -20
- data/lib/log_struct/version.rb +1 -1
- data/lib/log_struct.rb +6 -4
- data/logstruct.gemspec +19 -10
- metadata +63 -11
- data/lib/log_struct/builders/active_job.rb +0 -84
- data/lib/log_struct/integrations/good_job/log_subscriber.rb +0 -179
- data/lib/log_struct/integrations/rack.rb +0 -31
|
@@ -24,9 +24,10 @@ module LogStruct
|
|
|
24
24
|
# - Smart filtering reduces log volume for repetitive queries
|
|
25
25
|
#
|
|
26
26
|
# ## Security:
|
|
27
|
-
# -
|
|
28
|
-
#
|
|
29
|
-
# -
|
|
27
|
+
# - Bind parameters are filtered by their column names, like request params, and values that
|
|
28
|
+
# look like tokens or hashes are filtered too
|
|
29
|
+
# - The SQL is scrubbed like any string (emails, card numbers), but it isn't filtered: it can
|
|
30
|
+
# hold literal values (string conditions, or every value when prepared_statements is off)
|
|
30
31
|
#
|
|
31
32
|
# ## Configuration:
|
|
32
33
|
# ```ruby
|
|
@@ -55,7 +56,7 @@ module LogStruct
|
|
|
55
56
|
# is enabled. We still receive notifications via ActiveSupport.
|
|
56
57
|
if defined?(::ActiveRecord::LogSubscriber)
|
|
57
58
|
begin
|
|
58
|
-
::ActiveRecord::LogSubscriber
|
|
59
|
+
RailsLogSubscribers.detach(::ActiveRecord::LogSubscriber, :active_record)
|
|
59
60
|
rescue => e
|
|
60
61
|
LogStruct.handle_exception(e, source: LogStruct::Source::Internal)
|
|
61
62
|
end
|
|
@@ -63,9 +64,8 @@ module LogStruct
|
|
|
63
64
|
|
|
64
65
|
# Disable verbose query logs ("↳ caller") since LogStruct provides
|
|
65
66
|
# structured context and these lines are noisy/unstructured.
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
end
|
|
67
|
+
# (A setting on the ActiveRecord module, so ActiveRecord::Base isn't loaded early.)
|
|
68
|
+
::ActiveRecord.verbose_query_logs = false
|
|
69
69
|
|
|
70
70
|
subscribe_to_sql_notifications
|
|
71
71
|
true
|
|
@@ -175,14 +175,35 @@ module LogStruct
|
|
|
175
175
|
def self.extract_and_filter_binds(payload)
|
|
176
176
|
return nil unless LogStruct.config.integrations.sql_log_bind_params
|
|
177
177
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
178
|
+
binds = payload[:binds]
|
|
179
|
+
# The values as the database gets them, which Rails passes as a lambda since 6.0
|
|
180
|
+
casted_binds = payload[:type_casted_binds]
|
|
181
|
+
casted_binds = casted_binds.call if casted_binds.respond_to?(:call)
|
|
182
|
+
return nil unless binds.is_a?(Array) || casted_binds.is_a?(Array)
|
|
183
|
+
|
|
184
|
+
attributes = binds.is_a?(Array) ? binds : []
|
|
185
|
+
values = casted_binds.is_a?(Array) ? casted_binds : attributes
|
|
186
|
+
values.each_with_index.map do |value, index|
|
|
187
|
+
filter_bind(attributes[index], value)
|
|
188
|
+
end
|
|
189
|
+
end
|
|
181
190
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
191
|
+
# A bind's value, filtered by its column name like Rails' own SQL log lines
|
|
192
|
+
# (ActiveRecord::LogSubscriber#render_bind)
|
|
193
|
+
sig { params(attribute: T.untyped, value: T.untyped).returns(T.untyped) }
|
|
194
|
+
def self.filter_bind(attribute, value)
|
|
195
|
+
name = if attribute.is_a?(Array)
|
|
196
|
+
attribute.first&.name
|
|
197
|
+
elsif attribute.respond_to?(:name)
|
|
198
|
+
attribute.name
|
|
185
199
|
end
|
|
200
|
+
value = attribute.value_for_database if value.equal?(attribute) && attribute.respond_to?(:value_for_database)
|
|
201
|
+
if attribute.respond_to?(:type) && attribute.type.respond_to?(:binary?) && attribute.type.binary? && !value.nil?
|
|
202
|
+
return "<#{value.to_s.bytesize} bytes of binary data>"
|
|
203
|
+
end
|
|
204
|
+
return "[FILTERED]" if name && ParamFilters.should_filter_key?(name.to_s, value)
|
|
205
|
+
|
|
206
|
+
filter_bind_parameter(value)
|
|
186
207
|
end
|
|
187
208
|
|
|
188
209
|
# Extract database name from connection
|
|
@@ -270,16 +291,23 @@ module LogStruct
|
|
|
270
291
|
end
|
|
271
292
|
end
|
|
272
293
|
|
|
273
|
-
#
|
|
294
|
+
# Whether a String value is shaped like a secret: a token, a digest or a Bearer header. Values are
|
|
295
|
+
# filtered by their column names first; this catches binds without one. Text that mentions a
|
|
296
|
+
# password or a key isn't a secret.
|
|
274
297
|
sig { params(value: String).returns(T::Boolean) }
|
|
275
298
|
def self.looks_sensitive?(value)
|
|
276
|
-
#
|
|
277
|
-
return true if value.length > 50
|
|
299
|
+
# A long value with no spaces, which might be a token
|
|
300
|
+
return true if value.length > 50 && !value.match?(/\s/)
|
|
278
301
|
|
|
279
302
|
# Filter strings that look like hashed passwords, API keys, tokens
|
|
280
303
|
return true if value.match?(/\A[a-f0-9]{32,}\z/i) # MD5, SHA, etc.
|
|
281
304
|
return true if value.match?(/\A[A-Za-z0-9+\/]{20,}={0,2}\z/) # Base64
|
|
282
|
-
|
|
305
|
+
|
|
306
|
+
# Filter JWT tokens (header.payload.signature format, starts with "ey")
|
|
307
|
+
return true if value.match?(/\Aey[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\z/)
|
|
308
|
+
|
|
309
|
+
# Filter Bearer tokens
|
|
310
|
+
return true if value.match?(/\ABearer\s+/i)
|
|
283
311
|
|
|
284
312
|
false
|
|
285
313
|
end
|
|
@@ -19,14 +19,29 @@ module LogStruct
|
|
|
19
19
|
return nil unless config.enabled
|
|
20
20
|
return nil unless config.integrations.enable_activestorage
|
|
21
21
|
|
|
22
|
+
detach_rails_log_subscriber
|
|
23
|
+
|
|
22
24
|
# Subscribe to all ActiveStorage service events
|
|
23
25
|
::ActiveSupport::Notifications.subscribe(/service_.*\.active_storage/) do |*args|
|
|
24
26
|
process_active_storage_event(::ActiveSupport::Notifications::Event.new(*args), config)
|
|
27
|
+
rescue => e
|
|
28
|
+
# A problem with a line mustn't fail the upload or download it describes
|
|
29
|
+
LogStruct.handle_exception(e, source: Source::Internal, context: {integration: :active_storage})
|
|
25
30
|
end
|
|
26
31
|
|
|
27
32
|
true
|
|
28
33
|
end
|
|
29
34
|
|
|
35
|
+
# LogStruct's lines replace Rails' own ActiveStorage lines. Rails loads its log subscriber with
|
|
36
|
+
# the storage service, after LogStruct is set up, so it's loaded here to detach it.
|
|
37
|
+
sig { void }
|
|
38
|
+
def self.detach_rails_log_subscriber
|
|
39
|
+
require "active_storage/log_subscriber"
|
|
40
|
+
RailsLogSubscribers.detach(::ActiveStorage::LogSubscriber, :active_storage)
|
|
41
|
+
rescue LoadError
|
|
42
|
+
nil
|
|
43
|
+
end
|
|
44
|
+
|
|
30
45
|
private_class_method
|
|
31
46
|
|
|
32
47
|
# Process ActiveStorage events and create structured logs
|
|
@@ -54,19 +69,15 @@ module LogStruct
|
|
|
54
69
|
Event::Exist
|
|
55
70
|
when "service_url"
|
|
56
71
|
Event::Url
|
|
57
|
-
when "service_download_chunk"
|
|
72
|
+
when "service_download_chunk", "service_streaming_download"
|
|
58
73
|
Event::Download
|
|
59
|
-
when "service_stream"
|
|
60
|
-
Event::Stream
|
|
61
74
|
when "service_update_metadata"
|
|
62
75
|
Event::Metadata
|
|
63
76
|
else
|
|
64
|
-
|
|
77
|
+
# Events with no line of their own, such as service_mirror
|
|
78
|
+
return
|
|
65
79
|
end
|
|
66
80
|
|
|
67
|
-
# Map the event name to an operation
|
|
68
|
-
event_name.sub(/^service_/, "").to_sym
|
|
69
|
-
|
|
70
81
|
# Create structured log event using generated classes
|
|
71
82
|
log_data = case event_type
|
|
72
83
|
when Event::Upload
|
|
@@ -74,30 +85,27 @@ module LogStruct
|
|
|
74
85
|
storage: service_name.to_sym,
|
|
75
86
|
file_id: event.payload[:key]&.to_s,
|
|
76
87
|
checksum: event.payload[:checksum]&.to_s,
|
|
77
|
-
duration_ms: duration_ms
|
|
78
|
-
metadata: event.payload[:metadata],
|
|
79
|
-
filename: event.payload[:filename],
|
|
80
|
-
mime_type: event.payload[:content_type],
|
|
81
|
-
size: event.payload[:byte_size]
|
|
88
|
+
duration_ms: duration_ms
|
|
82
89
|
)
|
|
83
90
|
when Event::Download
|
|
84
91
|
Log::ActiveStorage::Download.new(
|
|
85
92
|
storage: service_name.to_sym,
|
|
86
93
|
file_id: event.payload[:key]&.to_s,
|
|
87
|
-
filename: event.payload[:filename],
|
|
88
94
|
range: event.payload[:range],
|
|
89
95
|
duration_ms: duration_ms
|
|
90
96
|
)
|
|
91
97
|
when Event::Delete
|
|
98
|
+
# delete_prefixed has a prefix instead of a key
|
|
92
99
|
Log::ActiveStorage::Delete.new(
|
|
93
100
|
storage: service_name.to_sym,
|
|
94
|
-
file_id: event.payload[:key]&.to_s
|
|
101
|
+
file_id: event.payload[:key]&.to_s,
|
|
102
|
+
prefix: event.payload[:prefix]&.to_s
|
|
95
103
|
)
|
|
96
104
|
when Event::Metadata
|
|
97
105
|
Log::ActiveStorage::Metadata.new(
|
|
98
106
|
storage: service_name.to_sym,
|
|
99
107
|
file_id: event.payload[:key]&.to_s,
|
|
100
|
-
|
|
108
|
+
mime_type: event.payload[:content_type]&.to_s
|
|
101
109
|
)
|
|
102
110
|
when Event::Exist
|
|
103
111
|
Log::ActiveStorage::Exist.new(
|
|
@@ -105,28 +113,20 @@ module LogStruct
|
|
|
105
113
|
file_id: event.payload[:key]&.to_s,
|
|
106
114
|
exist: event.payload[:exist]
|
|
107
115
|
)
|
|
108
|
-
when Event::Stream
|
|
109
|
-
Log::ActiveStorage::Stream.new(
|
|
110
|
-
storage: service_name.to_sym,
|
|
111
|
-
file_id: event.payload[:key]&.to_s,
|
|
112
|
-
prefix: event.payload[:prefix]
|
|
113
|
-
)
|
|
114
116
|
when Event::Url
|
|
117
|
+
# Without the URL: it's signed, and anyone who has it can read the file until it expires
|
|
115
118
|
Log::ActiveStorage::Url.new(
|
|
116
119
|
storage: service_name.to_sym,
|
|
117
|
-
file_id: event.payload[:key]&.to_s
|
|
118
|
-
url: event.payload[:url]
|
|
119
|
-
)
|
|
120
|
-
else
|
|
121
|
-
Log::ActiveStorage::Metadata.new(
|
|
122
|
-
storage: service_name.to_sym,
|
|
123
|
-
file_id: event.payload[:key]&.to_s,
|
|
124
|
-
metadata: event.payload[:metadata]
|
|
120
|
+
file_id: event.payload[:key]&.to_s
|
|
125
121
|
)
|
|
126
122
|
end
|
|
127
123
|
|
|
128
|
-
#
|
|
129
|
-
|
|
124
|
+
# Rails logs existence checks and URLs at debug level: an app generates a URL for each file it shows
|
|
125
|
+
if event_type == Event::Exist || event_type == Event::Url
|
|
126
|
+
LogStruct.debug(log_data)
|
|
127
|
+
else
|
|
128
|
+
LogStruct.info(log_data)
|
|
129
|
+
end
|
|
130
130
|
end
|
|
131
131
|
end
|
|
132
132
|
end
|
|
@@ -16,38 +16,42 @@ module LogStruct
|
|
|
16
16
|
mod = Module.new do
|
|
17
17
|
extend T::Sig
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
# Takes whatever Ahoy's track takes and passes it on as given, so Ahoy's defaults still apply
|
|
20
|
+
sig { params(name: T.untyped, args: T.untyped).returns(T.untyped) }
|
|
21
|
+
def track(name, *args)
|
|
21
22
|
result = super
|
|
22
23
|
begin
|
|
23
|
-
# Emit a lightweight structured log about the analytics event
|
|
24
|
-
data = {
|
|
25
|
-
ahoy_event: T.let(name, T.untyped)
|
|
26
|
-
}
|
|
27
|
-
data[:properties] = properties if properties
|
|
28
24
|
LogStruct.info(
|
|
29
25
|
LogStruct::Log::Ahoy.new(
|
|
30
26
|
message: "ahoy.track",
|
|
31
|
-
ahoy_event:
|
|
32
|
-
properties:
|
|
33
|
-
properties && properties.transform_keys { |k| k.to_sym },
|
|
34
|
-
T.nilable(T::Hash[Symbol, T.untyped])
|
|
35
|
-
)
|
|
27
|
+
ahoy_event: name.to_s,
|
|
28
|
+
properties: Integrations::Ahoy.properties_for_log(args.first)
|
|
36
29
|
)
|
|
37
30
|
)
|
|
38
31
|
rescue => e
|
|
39
|
-
#
|
|
40
|
-
LogStruct.handle_exception(e, source: LogStruct::Source::
|
|
32
|
+
# A problem with the log line is LogStruct's, and never fails the app's tracking call in production
|
|
33
|
+
LogStruct.handle_exception(e, source: LogStruct::Source::Internal, context: {integration: :ahoy})
|
|
41
34
|
end
|
|
42
35
|
result
|
|
43
36
|
end
|
|
44
37
|
end
|
|
45
38
|
|
|
46
|
-
|
|
39
|
+
::Ahoy::Tracker.prepend(mod)
|
|
47
40
|
end
|
|
48
41
|
|
|
49
42
|
true
|
|
50
43
|
end
|
|
44
|
+
|
|
45
|
+
# Event properties can come from the browser, so they can be anything; a non-Hash is logged as its class
|
|
46
|
+
sig { params(properties: T.untyped).returns(T.nilable(T::Hash[Symbol, T.untyped])) }
|
|
47
|
+
def self.properties_for_log(properties)
|
|
48
|
+
properties = properties.to_unsafe_h if properties.respond_to?(:to_unsafe_h)
|
|
49
|
+
case properties
|
|
50
|
+
when nil then nil
|
|
51
|
+
when Hash then properties.to_h { |key, value| [key.to_s.to_sym, value] }
|
|
52
|
+
else {_class: properties.class.name}
|
|
53
|
+
end
|
|
54
|
+
end
|
|
51
55
|
end
|
|
52
56
|
end
|
|
53
57
|
end
|
|
@@ -27,78 +27,82 @@ module LogStruct
|
|
|
27
27
|
true
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
-
#
|
|
30
|
+
# Logs the operations that store, delete or fetch a file: store! (an upload, when there's a
|
|
31
|
+
# cached file to store), remove! (a delete) and download! (a remote URL fetched into the cache).
|
|
32
|
+
# retrieve_from_store! isn't logged: it only points the uploader at a stored file, and runs
|
|
33
|
+
# each time a record's mounted file is read.
|
|
31
34
|
module LoggingMethods
|
|
32
35
|
extend T::Sig
|
|
33
36
|
extend T::Helpers
|
|
34
37
|
requires_ancestor { ::CarrierWave::Uploader::Base }
|
|
35
38
|
|
|
36
|
-
# Log file storage operations
|
|
37
39
|
sig { params(args: T.untyped).returns(T.untyped) }
|
|
38
40
|
def store!(*args)
|
|
41
|
+
# CarrierWave only stores a cached file; a record's save calls store! on every uploader
|
|
42
|
+
storing = !cache_only && (cached? || !args.first.nil?)
|
|
39
43
|
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
40
44
|
result = super
|
|
41
|
-
|
|
45
|
+
return result unless storing && !cached?
|
|
42
46
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
storage: storage.class.name.split("::").last.downcase.to_sym,
|
|
57
|
-
file_id: identifier,
|
|
58
|
-
filename: file.filename,
|
|
59
|
-
mime_type: file.content_type,
|
|
60
|
-
size: file_size,
|
|
61
|
-
duration_ms: (duration * 1000.0).to_f,
|
|
62
|
-
uploader: self.class.name,
|
|
63
|
-
model: model.class.name,
|
|
64
|
-
mount_point: mounted_as.to_s,
|
|
65
|
-
version: version_name.to_s,
|
|
66
|
-
store_path: store_path,
|
|
67
|
-
extension: file.extension
|
|
68
|
-
)
|
|
47
|
+
duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0
|
|
48
|
+
log_file_operation do
|
|
49
|
+
# file_fields is a Hash Sorbet can't check against the struct's keyword arguments
|
|
50
|
+
Log::CarrierWave::Upload.new(
|
|
51
|
+
**T.unsafe(file_fields),
|
|
52
|
+
filename: file&.filename,
|
|
53
|
+
mime_type: file&.content_type,
|
|
54
|
+
size: file&.size,
|
|
55
|
+
duration_ms: duration_ms
|
|
56
|
+
)
|
|
57
|
+
end
|
|
58
|
+
result
|
|
59
|
+
end
|
|
69
60
|
|
|
70
|
-
|
|
61
|
+
sig { returns(T.untyped) }
|
|
62
|
+
def remove!
|
|
63
|
+
fields = file ? file_fields : nil
|
|
64
|
+
result = super
|
|
65
|
+
log_file_operation { Log::CarrierWave::Delete.new(**T.unsafe(fields)) } if fields
|
|
71
66
|
result
|
|
72
67
|
end
|
|
73
68
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
def retrieve_from_store!(identifier, *args)
|
|
77
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
69
|
+
sig { params(uri: T.untyped, remote_headers: T.untyped).returns(T.untyped) }
|
|
70
|
+
def download!(uri, remote_headers = {})
|
|
78
71
|
result = super
|
|
79
|
-
|
|
72
|
+
log_file_operation do
|
|
73
|
+
Log::CarrierWave::Download.new(
|
|
74
|
+
**T.unsafe(file_fields),
|
|
75
|
+
filename: file&.filename,
|
|
76
|
+
mime_type: file&.content_type,
|
|
77
|
+
size: file&.size
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
result
|
|
81
|
+
end
|
|
80
82
|
|
|
81
|
-
|
|
82
|
-
file_size = file.size if file&.respond_to?(:size)
|
|
83
|
+
private
|
|
83
84
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
size: file_size,
|
|
91
|
-
# No duration field on Download event schema
|
|
85
|
+
# The fields every CarrierWave line has
|
|
86
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
87
|
+
def file_fields
|
|
88
|
+
{
|
|
89
|
+
storage: storage.class.name.to_s.split("::").last.to_s.downcase.to_sym,
|
|
90
|
+
file_id: identifier.to_s,
|
|
92
91
|
uploader: self.class.name,
|
|
93
|
-
model: model
|
|
94
|
-
mount_point: mounted_as
|
|
95
|
-
version: version_name
|
|
92
|
+
model: model&.class&.name,
|
|
93
|
+
mount_point: mounted_as&.to_s,
|
|
94
|
+
version: version_name&.to_s,
|
|
96
95
|
store_path: store_path,
|
|
97
96
|
extension: file&.extension
|
|
98
|
-
|
|
97
|
+
}
|
|
98
|
+
end
|
|
99
99
|
|
|
100
|
-
|
|
101
|
-
|
|
100
|
+
# A problem with a log line mustn't stop the file operation that has already happened
|
|
101
|
+
sig { params(block: T.proc.returns(Log::Interfaces::CommonFields)).void }
|
|
102
|
+
def log_file_operation(&block)
|
|
103
|
+
LogStruct.info(yield)
|
|
104
|
+
rescue => e
|
|
105
|
+
LogStruct.handle_exception(e, source: Source::Internal, context: {integration: :carrierwave})
|
|
102
106
|
end
|
|
103
107
|
end
|
|
104
108
|
end
|
|
@@ -48,6 +48,12 @@ module LogStruct
|
|
|
48
48
|
end
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
+
# A problem with a Dotenv log line is LogStruct's own: raised in development and test, logged in production
|
|
52
|
+
sig { params(error: StandardError).void }
|
|
53
|
+
def handle_log_error(error)
|
|
54
|
+
LogStruct.handle_exception(error, source: LogStruct::Source::Internal, context: {integration: :dotenv})
|
|
55
|
+
end
|
|
56
|
+
|
|
51
57
|
sig { void }
|
|
52
58
|
def subscribe!
|
|
53
59
|
# Guard against double subscription
|
|
@@ -66,11 +72,7 @@ module LogStruct
|
|
|
66
72
|
ts = event.time ? Time.at(event.time) : Time.now
|
|
67
73
|
LogStruct.info(Log::Dotenv::Load.new(file: file, timestamp: ts))
|
|
68
74
|
rescue => e
|
|
69
|
-
|
|
70
|
-
raise
|
|
71
|
-
else
|
|
72
|
-
LogStruct.handle_exception(e, source: LogStruct::Source::Dotenv)
|
|
73
|
-
end
|
|
75
|
+
handle_log_error(e)
|
|
74
76
|
end
|
|
75
77
|
|
|
76
78
|
instrumenter.subscribe("update.dotenv") do |*args|
|
|
@@ -82,11 +84,7 @@ module LogStruct
|
|
|
82
84
|
ts = event.time ? Time.at(event.time) : Time.now
|
|
83
85
|
LogStruct.debug(Log::Dotenv::Update.new(vars: vars, timestamp: ts))
|
|
84
86
|
rescue => e
|
|
85
|
-
|
|
86
|
-
raise
|
|
87
|
-
else
|
|
88
|
-
LogStruct.handle_exception(e, source: LogStruct::Source::Dotenv)
|
|
89
|
-
end
|
|
87
|
+
handle_log_error(e)
|
|
90
88
|
end
|
|
91
89
|
|
|
92
90
|
instrumenter.subscribe("save.dotenv") do |*args|
|
|
@@ -95,11 +93,7 @@ module LogStruct
|
|
|
95
93
|
ts = event.time ? Time.at(event.time) : Time.now
|
|
96
94
|
LogStruct.info(Log::Dotenv::Save.new(snapshot: true, timestamp: ts))
|
|
97
95
|
rescue => e
|
|
98
|
-
|
|
99
|
-
raise
|
|
100
|
-
else
|
|
101
|
-
LogStruct.handle_exception(e, source: LogStruct::Source::Dotenv)
|
|
102
|
-
end
|
|
96
|
+
handle_log_error(e)
|
|
103
97
|
end
|
|
104
98
|
|
|
105
99
|
instrumenter.subscribe("restore.dotenv") do |*args|
|
|
@@ -111,11 +105,7 @@ module LogStruct
|
|
|
111
105
|
ts = event.time ? Time.at(event.time) : Time.now
|
|
112
106
|
LogStruct.info(Log::Dotenv::Restore.new(vars: vars, timestamp: ts))
|
|
113
107
|
rescue => e
|
|
114
|
-
|
|
115
|
-
raise
|
|
116
|
-
else
|
|
117
|
-
LogStruct.handle_exception(e, source: LogStruct::Source::Dotenv)
|
|
118
|
-
end
|
|
108
|
+
handle_log_error(e)
|
|
119
109
|
end
|
|
120
110
|
|
|
121
111
|
STATE.subscribed = true
|
|
@@ -130,8 +120,8 @@ module LogStruct
|
|
|
130
120
|
return unless defined?(::ActiveSupport::Notifications)
|
|
131
121
|
|
|
132
122
|
instrumenter = if Object.const_defined?(:Dotenv)
|
|
133
|
-
dm =
|
|
134
|
-
dm.respond_to?(:instrumenter) ?
|
|
123
|
+
dm = Object.const_get(:Dotenv)
|
|
124
|
+
dm.respond_to?(:instrumenter) ? dm.instrumenter : ::ActiveSupport::Notifications
|
|
135
125
|
else
|
|
136
126
|
::ActiveSupport::Notifications
|
|
137
127
|
end
|
|
@@ -143,7 +133,7 @@ module LogStruct
|
|
|
143
133
|
ts = event.time ? Time.at(event.time) : Time.now
|
|
144
134
|
LogStruct::BootBuffer.add(Log::Dotenv::Load.new(file: file, timestamp: ts))
|
|
145
135
|
rescue => e
|
|
146
|
-
|
|
136
|
+
handle_log_error(e)
|
|
147
137
|
end
|
|
148
138
|
|
|
149
139
|
instrumenter.subscribe("update.dotenv") do |*args|
|
|
@@ -153,7 +143,7 @@ module LogStruct
|
|
|
153
143
|
ts = event.time ? Time.at(event.time) : Time.now
|
|
154
144
|
LogStruct::BootBuffer.add(Log::Dotenv::Update.new(vars: vars, timestamp: ts))
|
|
155
145
|
rescue => e
|
|
156
|
-
|
|
146
|
+
handle_log_error(e)
|
|
157
147
|
end
|
|
158
148
|
|
|
159
149
|
instrumenter.subscribe("save.dotenv") do |*args|
|
|
@@ -161,7 +151,7 @@ module LogStruct
|
|
|
161
151
|
ts = event.time ? Time.at(event.time) : Time.now
|
|
162
152
|
LogStruct::BootBuffer.add(Log::Dotenv::Save.new(snapshot: true, timestamp: ts))
|
|
163
153
|
rescue => e
|
|
164
|
-
|
|
154
|
+
handle_log_error(e)
|
|
165
155
|
end
|
|
166
156
|
|
|
167
157
|
instrumenter.subscribe("restore.dotenv") do |*args|
|
|
@@ -171,7 +161,7 @@ module LogStruct
|
|
|
171
161
|
ts = event.time ? Time.at(event.time) : Time.now
|
|
172
162
|
LogStruct::BootBuffer.add(Log::Dotenv::Restore.new(vars: vars, timestamp: ts))
|
|
173
163
|
rescue => e
|
|
174
|
-
|
|
164
|
+
handle_log_error(e)
|
|
175
165
|
end
|
|
176
166
|
|
|
177
167
|
@@boot_subscribed = true
|
|
@@ -183,9 +173,9 @@ module LogStruct
|
|
|
183
173
|
return unless Object.const_defined?(:Dotenv)
|
|
184
174
|
# Do not intercept when LogStruct is disabled; allow original dotenv replay
|
|
185
175
|
return unless LogStruct.enabled?
|
|
186
|
-
dotenv_mod =
|
|
176
|
+
dotenv_mod = Object.const_get(:Dotenv)
|
|
187
177
|
return unless dotenv_mod.const_defined?(:Rails)
|
|
188
|
-
klass =
|
|
178
|
+
klass = dotenv_mod.const_get(:Rails)
|
|
189
179
|
return if klass.instance_variable_defined?(:@_logstruct_replay_patched)
|
|
190
180
|
|
|
191
181
|
original = klass.instance_method(:logger=)
|
|
@@ -212,15 +202,15 @@ module LogStruct
|
|
|
212
202
|
def self.resolve_boot_logs!
|
|
213
203
|
# If LogStruct is disabled, do not alter dotenv behavior at all
|
|
214
204
|
return unless LogStruct.enabled?
|
|
215
|
-
dotenv_mod = Object.const_defined?(:Dotenv) ?
|
|
216
|
-
klass = dotenv_mod&.const_defined?(:Rails) ?
|
|
205
|
+
dotenv_mod = Object.const_defined?(:Dotenv) ? Object.const_get(:Dotenv) : nil
|
|
206
|
+
klass = dotenv_mod&.const_defined?(:Rails) ? dotenv_mod.const_get(:Rails) : nil
|
|
217
207
|
|
|
218
208
|
pending_logger = nil
|
|
219
209
|
railtie_instance = nil
|
|
220
210
|
if klass&.respond_to?(:instance)
|
|
221
211
|
railtie_instance = klass.instance
|
|
222
212
|
if railtie_instance.respond_to?(:logstruct_pending_dotenv_logger)
|
|
223
|
-
pending_logger =
|
|
213
|
+
pending_logger = railtie_instance.logstruct_pending_dotenv_logger
|
|
224
214
|
end
|
|
225
215
|
end
|
|
226
216
|
|
|
@@ -242,7 +232,7 @@ module LogStruct
|
|
|
242
232
|
|
|
243
233
|
# Detach original subscriber and subscribe runtime structured
|
|
244
234
|
if dotenv_mod&.const_defined?(:LogSubscriber)
|
|
245
|
-
|
|
235
|
+
dotenv_mod.const_get(:LogSubscriber).detach_from(:dotenv)
|
|
246
236
|
end
|
|
247
237
|
LogStruct::Integrations::Dotenv.subscribe!
|
|
248
238
|
|
|
@@ -10,16 +10,8 @@ module LogStruct
|
|
|
10
10
|
module GoodJob
|
|
11
11
|
# Custom Logger for GoodJob that creates LogStruct::Log::GoodJob entries
|
|
12
12
|
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
# operations and events.
|
|
16
|
-
#
|
|
17
|
-
# ## Benefits:
|
|
18
|
-
# - High-performance logging with SemanticLogger backend
|
|
19
|
-
# - Structured GoodJob-specific log entries
|
|
20
|
-
# - Automatic job context capture
|
|
21
|
-
# - Thread and process information
|
|
22
|
-
# - Performance metrics and timing data
|
|
13
|
+
# GoodJob's own messages are logged as Log::GoodJob::Log lines with the process and thread,
|
|
14
|
+
# and, during a job, the job's ID, class, queue, execution count, scheduled time and priority.
|
|
23
15
|
#
|
|
24
16
|
# ## Usage:
|
|
25
17
|
# This logger is automatically configured when the GoodJob integration
|
|
@@ -30,22 +22,15 @@ module LogStruct
|
|
|
30
22
|
|
|
31
23
|
# Override log methods to create GoodJob-specific log structs
|
|
32
24
|
%i[debug info warn error fatal].each do |level|
|
|
33
|
-
define_method(level) do |message = nil, payload = nil, &block|
|
|
34
|
-
#
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
execution = Thread.current[:good_job_execution]
|
|
39
|
-
if execution.respond_to?(:job_id)
|
|
40
|
-
job_context[:job_id] = execution.job_id
|
|
41
|
-
job_context[:job_class] = execution.job_class if execution.respond_to?(:job_class)
|
|
42
|
-
job_context[:queue_name] = execution.queue_name if execution.respond_to?(:queue_name)
|
|
43
|
-
job_context[:executions] = execution.executions if execution.respond_to?(:executions)
|
|
44
|
-
job_context[:scheduled_at] = execution.scheduled_at if execution.respond_to?(:scheduled_at)
|
|
45
|
-
job_context[:priority] = execution.priority if execution.respond_to?(:priority)
|
|
46
|
-
end
|
|
25
|
+
define_method(level) do |message = nil, payload = nil, exception = nil, &block|
|
|
26
|
+
# LogStruct log entries and Hashes (like the thread error handler's) are logged as they are, and a
|
|
27
|
+
# message with an exception is an error line with both
|
|
28
|
+
if !(message.nil? || message.is_a?(String)) || exception || payload.is_a?(Exception)
|
|
29
|
+
next super(message, payload, exception, &block)
|
|
47
30
|
end
|
|
48
31
|
|
|
32
|
+
job_context = Logger.current_job_context
|
|
33
|
+
|
|
49
34
|
log_struct = Log::GoodJob::Log.new(
|
|
50
35
|
message: message || (block ? block.call : ""),
|
|
51
36
|
process_id: ::Process.pid,
|
|
@@ -61,6 +46,27 @@ module LogStruct
|
|
|
61
46
|
super(log_struct, payload, &nil)
|
|
62
47
|
end
|
|
63
48
|
end
|
|
49
|
+
|
|
50
|
+
# The job GoodJob is running on this thread: CurrentThread.job (a GoodJob::Job) in GoodJob 4,
|
|
51
|
+
# CurrentThread.execution in GoodJob 3
|
|
52
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
53
|
+
def self.current_job_context
|
|
54
|
+
return {} unless defined?(::GoodJob::CurrentThread)
|
|
55
|
+
|
|
56
|
+
current = ::GoodJob::CurrentThread
|
|
57
|
+
job = (current.job if current.respond_to?(:job)) || (current.execution if current.respond_to?(:execution))
|
|
58
|
+
return {} unless job
|
|
59
|
+
|
|
60
|
+
job_class = job.try(:job_class) || job.try(:serialized_params)&.dig("job_class")
|
|
61
|
+
{
|
|
62
|
+
job_id: job.try(:active_job_id)&.to_s,
|
|
63
|
+
job_class: job_class&.to_s,
|
|
64
|
+
queue_name: job.try(:queue_name)&.to_sym,
|
|
65
|
+
executions: job.try(:executions_count) || job.try(:executions),
|
|
66
|
+
scheduled_at: job.try(:scheduled_at),
|
|
67
|
+
priority: job.try(:priority)
|
|
68
|
+
}
|
|
69
|
+
end
|
|
64
70
|
end
|
|
65
71
|
end
|
|
66
72
|
end
|