canonical_log 0.1.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/canonical_log/configuration.rb +80 -10
- data/lib/canonical_log/emitter.rb +41 -0
- data/lib/canonical_log/event.rb +8 -4
- data/lib/canonical_log/formatters/logfmt.rb +42 -0
- data/lib/canonical_log/formatters/pretty.rb +39 -0
- data/lib/canonical_log/integrations/error_enrichment.rb +1 -1
- data/lib/canonical_log/integrations/sidekiq.rb +3 -13
- data/lib/canonical_log/middleware.rb +51 -46
- data/lib/canonical_log/rails_log_suppressor.rb +46 -0
- data/lib/canonical_log/railtie.rb +6 -0
- data/lib/canonical_log/sampling.rb +2 -6
- data/lib/canonical_log/sinks/base.rb +1 -1
- data/lib/canonical_log/sinks/null.rb +11 -0
- data/lib/canonical_log/sinks/rails_logger.rb +2 -2
- data/lib/canonical_log/sinks/stdout.rb +1 -1
- data/lib/canonical_log/sql_sanitizer.rb +18 -0
- data/lib/canonical_log/subscribers/action_controller.rb +20 -6
- data/lib/canonical_log/subscribers/active_job.rb +45 -0
- data/lib/canonical_log/subscribers/active_record.rb +18 -7
- data/lib/canonical_log/subscribers/active_support_cache.rb +46 -0
- data/lib/canonical_log/version.rb +1 -1
- data/lib/canonical_log.rb +8 -0
- data/lib/generators/canonical_log/templates/canonical_log.rb +45 -18
- metadata +9 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0f2b86d164c19175dd3f83d60146ab61f4e31d382feb35033a7e616d07d785ff
|
|
4
|
+
data.tar.gz: 2539db431fc881bf3dc4da1b20788fc4a56bf0e512ec253975acd4bbbe019220
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f5959c2e8cb0d5cfe2cd4327c55d908719caeb289fdf2d93d446a22c598a7bee936fccade3c320d9f88a7da31834ff71cfc0ae3f192386e090c0cdf8c2e6fd35
|
|
7
|
+
data.tar.gz: f86c56cd4f7efc2c7e62144e94743f6fa812506eaa69ab1c9f02f00872b3eb69a82beb92d39a8aca444fdc205c97167bdf86f24aae73e950a0f61c36a8939781
|
|
@@ -1,21 +1,43 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'uri'
|
|
4
|
+
|
|
3
5
|
module CanonicalLog
|
|
4
6
|
class Configuration
|
|
5
7
|
attr_accessor :sinks, :param_filter_keys, :slow_query_threshold_ms,
|
|
6
8
|
:user_context, :before_emit, :ignored_paths,
|
|
7
|
-
:sample_rate, :slow_request_threshold_ms, :sampling
|
|
9
|
+
:sample_rate, :slow_request_threshold_ms, :sampling,
|
|
10
|
+
:enabled, :suppress_rails_logging, :format,
|
|
11
|
+
:filter_sql_literals, :filter_query_string,
|
|
12
|
+
:log_level_resolver, :default_fields,
|
|
13
|
+
:error_backtrace_lines
|
|
8
14
|
|
|
9
15
|
def initialize
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
set_defaults
|
|
17
|
+
set_filter_defaults
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def pretty=(value)
|
|
21
|
+
self.format = value ? :pretty : :json
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def pretty?
|
|
25
|
+
format == :pretty
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def resolve_log_level(event_hash)
|
|
29
|
+
if @log_level_resolver
|
|
30
|
+
@log_level_resolver.call(event_hash)
|
|
31
|
+
else
|
|
32
|
+
status = event_hash[:http_status].to_i
|
|
33
|
+
if event_hash[:error] || status >= 500
|
|
34
|
+
:error
|
|
35
|
+
elsif status >= 400
|
|
36
|
+
:warn
|
|
37
|
+
else
|
|
38
|
+
:info
|
|
39
|
+
end
|
|
40
|
+
end
|
|
19
41
|
end
|
|
20
42
|
|
|
21
43
|
def resolved_sinks
|
|
@@ -38,5 +60,53 @@ module CanonicalLog
|
|
|
38
60
|
Sampling.sample?(event_hash, self)
|
|
39
61
|
end
|
|
40
62
|
end
|
|
63
|
+
|
|
64
|
+
def ignored_path?(path)
|
|
65
|
+
@ignored_paths.any? do |pattern|
|
|
66
|
+
case pattern
|
|
67
|
+
when Regexp then pattern.match?(path)
|
|
68
|
+
when String then path.start_with?(pattern)
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def filtered_query(query)
|
|
74
|
+
params = URI.decode_www_form(query)
|
|
75
|
+
filtered = params.map do |key, value|
|
|
76
|
+
@param_filter_keys.include?(key) ? [key, '[FILTERED]'] : [key, value]
|
|
77
|
+
end
|
|
78
|
+
URI.encode_www_form(filtered)
|
|
79
|
+
rescue ArgumentError
|
|
80
|
+
query
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private
|
|
84
|
+
|
|
85
|
+
def set_defaults
|
|
86
|
+
@sinks = :auto
|
|
87
|
+
@user_context = nil
|
|
88
|
+
@before_emit = nil
|
|
89
|
+
@ignored_paths = []
|
|
90
|
+
@sample_rate = 1.0
|
|
91
|
+
@slow_request_threshold_ms = 2000.0
|
|
92
|
+
@slow_query_threshold_ms = 100.0
|
|
93
|
+
@sampling = nil
|
|
94
|
+
@enabled = defined?(Rails) ? Rails.env.production? : true
|
|
95
|
+
@suppress_rails_logging = false
|
|
96
|
+
@format = :json
|
|
97
|
+
@log_level_resolver = nil
|
|
98
|
+
@default_fields = {}
|
|
99
|
+
@error_backtrace_lines = 5
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def set_filter_defaults
|
|
103
|
+
@param_filter_keys = [
|
|
104
|
+
'password', 'password_confirmation', 'token', 'secret',
|
|
105
|
+
'secret_key', 'api_key', 'access_token', 'credit_card',
|
|
106
|
+
'card_number', 'cvv', 'ssn', 'authorization'
|
|
107
|
+
]
|
|
108
|
+
@filter_sql_literals = true
|
|
109
|
+
@filter_query_string = true
|
|
110
|
+
end
|
|
41
111
|
end
|
|
42
112
|
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CanonicalLog
|
|
4
|
+
module Emitter
|
|
5
|
+
def self.emit!(event, config = CanonicalLog.configuration)
|
|
6
|
+
config.before_emit&.call(event)
|
|
7
|
+
event_hash = config.default_fields.merge(event.to_h)
|
|
8
|
+
return unless config.should_sample?(event_hash)
|
|
9
|
+
|
|
10
|
+
level = config.resolve_log_level(event_hash)
|
|
11
|
+
event_hash[:level] = level.to_s
|
|
12
|
+
event_hash[:message] ||= build_message(event_hash)
|
|
13
|
+
write_to_sinks(event_hash, config, level: level)
|
|
14
|
+
rescue StandardError => e
|
|
15
|
+
warn "[CanonicalLog] Emit error: #{e.message}"
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.build_message(event_hash)
|
|
19
|
+
[event_hash[:http_method], event_hash[:path], event_hash[:http_status]].compact.join(' ')
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.write_to_sinks(event_hash, config, level: :info)
|
|
23
|
+
json = serialize(event_hash, config)
|
|
24
|
+
config.resolved_sinks.each do |sink|
|
|
25
|
+
sink.write(json, level: level)
|
|
26
|
+
rescue StandardError => e
|
|
27
|
+
warn "[CanonicalLog] Sink error (#{sink.class}): #{e.message}"
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def self.serialize(event_hash, config)
|
|
32
|
+
case config.format
|
|
33
|
+
when :pretty then CanonicalLog::Formatters::Pretty.format(event_hash)
|
|
34
|
+
when :logfmt then CanonicalLog::Formatters::Logfmt.format(event_hash)
|
|
35
|
+
else event_hash.to_json
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private_class_method :build_message, :write_to_sinks, :serialize
|
|
40
|
+
end
|
|
41
|
+
end
|
data/lib/canonical_log/event.rb
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'json'
|
|
4
|
+
require 'time'
|
|
4
5
|
|
|
5
6
|
module CanonicalLog
|
|
6
7
|
class Event
|
|
7
|
-
CATEGORIES =
|
|
8
|
+
CATEGORIES = [:user, :business, :infra, :service].freeze
|
|
8
9
|
|
|
9
10
|
def initialize
|
|
10
11
|
@fields = {}
|
|
@@ -51,10 +52,13 @@ module CanonicalLog
|
|
|
51
52
|
# Structured error capture
|
|
52
53
|
def add_error(error, metadata = {})
|
|
53
54
|
@mutex.synchronize do
|
|
54
|
-
|
|
55
|
+
backtrace_lines = CanonicalLog.configuration.error_backtrace_lines
|
|
56
|
+
error_hash = {
|
|
55
57
|
class: error.class.name,
|
|
56
|
-
message: error.message
|
|
57
|
-
}
|
|
58
|
+
message: error.message,
|
|
59
|
+
}
|
|
60
|
+
error_hash[:backtrace] = error.backtrace.first(backtrace_lines) if backtrace_lines.positive? && error.backtrace
|
|
61
|
+
@fields[:error] = error_hash.merge(metadata)
|
|
58
62
|
end
|
|
59
63
|
end
|
|
60
64
|
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CanonicalLog
|
|
4
|
+
module Formatters
|
|
5
|
+
module Logfmt
|
|
6
|
+
def self.format(hash)
|
|
7
|
+
flatten(hash).map { |k, v| "#{k}=#{format_value(v)}" }.join(' ')
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def self.flatten(hash, prefix = nil, result = {})
|
|
11
|
+
hash.each do |key, value|
|
|
12
|
+
full_key = prefix ? "#{prefix}.#{key}" : key.to_s
|
|
13
|
+
if value.is_a?(Hash)
|
|
14
|
+
flatten(value, full_key, result)
|
|
15
|
+
else
|
|
16
|
+
result[full_key] = value
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
result
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.format_value(value)
|
|
23
|
+
case value
|
|
24
|
+
when nil then ''
|
|
25
|
+
when true, false, Numeric then value.to_s
|
|
26
|
+
when Array then maybe_quote(value.join(','))
|
|
27
|
+
else maybe_quote(value.to_s)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def self.maybe_quote(str)
|
|
32
|
+
if str.empty? || str.match?(/[\s="\\]/)
|
|
33
|
+
"\"#{str.gsub('\\', '\\\\\\\\').gsub('"', '\\"')}\""
|
|
34
|
+
else
|
|
35
|
+
str
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private_class_method :flatten, :format_value, :maybe_quote
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module CanonicalLog
|
|
6
|
+
module Formatters
|
|
7
|
+
module Pretty
|
|
8
|
+
CYAN = "\e[36m"
|
|
9
|
+
GREEN = "\e[32m"
|
|
10
|
+
YELLOW = "\e[33m"
|
|
11
|
+
MAGENTA = "\e[35m"
|
|
12
|
+
GRAY = "\e[90m"
|
|
13
|
+
RESET = "\e[0m"
|
|
14
|
+
|
|
15
|
+
def self.format(hash)
|
|
16
|
+
json = JSON.pretty_generate(hash)
|
|
17
|
+
colorize(json)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.colorize(json)
|
|
21
|
+
json.gsub(/("(?:[^"\\]|\\.)*")(\s*:)?|(\b(?:true|false)\b)|\bnull\b|(-?\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b)/) do
|
|
22
|
+
if Regexp.last_match(2)
|
|
23
|
+
"#{CYAN}#{Regexp.last_match(1)}#{RESET}#{Regexp.last_match(2)}"
|
|
24
|
+
elsif Regexp.last_match(1)
|
|
25
|
+
"#{GREEN}#{Regexp.last_match(1)}#{RESET}"
|
|
26
|
+
elsif Regexp.last_match(3)
|
|
27
|
+
"#{MAGENTA}#{Regexp.last_match(3)}#{RESET}"
|
|
28
|
+
elsif Regexp.last_match(0) == 'null'
|
|
29
|
+
"#{GRAY}null#{RESET}"
|
|
30
|
+
else
|
|
31
|
+
"#{YELLOW}#{Regexp.last_match(0)}#{RESET}"
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private_class_method :colorize
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -9,14 +9,14 @@ module CanonicalLog
|
|
|
9
9
|
event.add(
|
|
10
10
|
job_class: msg['class'],
|
|
11
11
|
queue: queue,
|
|
12
|
-
jid: msg['jid']
|
|
12
|
+
jid: msg['jid'],
|
|
13
13
|
)
|
|
14
14
|
|
|
15
15
|
yield
|
|
16
16
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
17
17
|
event&.add(
|
|
18
18
|
error_class: e.class.name,
|
|
19
|
-
error_message: e.message
|
|
19
|
+
error_message: e.message,
|
|
20
20
|
)
|
|
21
21
|
raise
|
|
22
22
|
ensure
|
|
@@ -30,17 +30,7 @@ module CanonicalLog
|
|
|
30
30
|
event = Context.current
|
|
31
31
|
return unless event
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
config.before_emit&.call(event)
|
|
35
|
-
json = event.to_json
|
|
36
|
-
|
|
37
|
-
config.resolved_sinks.each do |sink|
|
|
38
|
-
sink.write(json)
|
|
39
|
-
rescue StandardError => e
|
|
40
|
-
warn "[CanonicalLog] Sink error (#{sink.class}): #{e.message}"
|
|
41
|
-
end
|
|
42
|
-
rescue StandardError => e
|
|
43
|
-
warn "[CanonicalLog] Emit error: #{e.message}"
|
|
33
|
+
Emitter.emit!(event)
|
|
44
34
|
end
|
|
45
35
|
end
|
|
46
36
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'uri'
|
|
4
|
+
|
|
3
5
|
module CanonicalLog
|
|
4
6
|
class Middleware
|
|
5
7
|
def initialize(app)
|
|
@@ -7,45 +9,73 @@ module CanonicalLog
|
|
|
7
9
|
end
|
|
8
10
|
|
|
9
11
|
def call(env)
|
|
10
|
-
return @app.call(env) if
|
|
12
|
+
return @app.call(env) if skip?(env)
|
|
11
13
|
|
|
12
14
|
Context.init!
|
|
13
15
|
seed_request_fields(env)
|
|
14
|
-
|
|
15
|
-
status, headers, body = @app.call(env)
|
|
16
|
-
Context.current&.set(:http_status, status)
|
|
17
|
-
enrich_user_context(env)
|
|
18
|
-
[status, headers, body]
|
|
16
|
+
execute_request(env)
|
|
19
17
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
20
18
|
Context.current&.add_error(e)
|
|
21
19
|
Context.current&.set(:http_status, 500)
|
|
22
20
|
raise
|
|
23
21
|
ensure
|
|
24
|
-
|
|
25
|
-
Context.clear!
|
|
22
|
+
finalize!
|
|
26
23
|
end
|
|
27
24
|
|
|
28
25
|
private
|
|
29
26
|
|
|
27
|
+
def skip?(env)
|
|
28
|
+
config = CanonicalLog.configuration
|
|
29
|
+
!config.enabled || config.ignored_path?(env['PATH_INFO'])
|
|
30
|
+
end
|
|
31
|
+
|
|
30
32
|
def seed_request_fields(env)
|
|
31
33
|
event = Context.current
|
|
32
34
|
return unless event
|
|
33
35
|
|
|
34
|
-
request_id = env['action_dispatch.request_id'] ||
|
|
35
|
-
env['HTTP_X_REQUEST_ID'] ||
|
|
36
|
-
SecureRandom.uuid
|
|
37
|
-
|
|
38
36
|
event.add(
|
|
39
|
-
request_id:
|
|
37
|
+
request_id: resolve_request_id(env),
|
|
40
38
|
http_method: env['REQUEST_METHOD'],
|
|
41
39
|
path: env['PATH_INFO'],
|
|
42
|
-
query_string: env
|
|
40
|
+
query_string: resolve_query_string(env),
|
|
43
41
|
remote_ip: env['HTTP_X_FORWARDED_FOR'] || env['REMOTE_ADDR'],
|
|
44
42
|
user_agent: env['HTTP_USER_AGENT'],
|
|
45
|
-
content_type: env['CONTENT_TYPE']
|
|
43
|
+
content_type: env['CONTENT_TYPE'],
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
enrich_trace_context(event)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def resolve_request_id(env)
|
|
50
|
+
env['action_dispatch.request_id'] || env['HTTP_X_REQUEST_ID'] || SecureRandom.uuid
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def resolve_query_string(env)
|
|
54
|
+
raw = env['QUERY_STRING'].to_s
|
|
55
|
+
return nil if raw.empty?
|
|
56
|
+
|
|
57
|
+
CanonicalLog.configuration.filter_query_string ? CanonicalLog.configuration.filtered_query(raw) : raw
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def enrich_trace_context(event)
|
|
61
|
+
return unless defined?(OpenTelemetry::Trace)
|
|
62
|
+
|
|
63
|
+
span_context = OpenTelemetry::Trace.current_span.context
|
|
64
|
+
return unless span_context.valid?
|
|
65
|
+
|
|
66
|
+
event.add(
|
|
67
|
+
trace_id: span_context.hex_trace_id,
|
|
68
|
+
span_id: span_context.hex_span_id,
|
|
46
69
|
)
|
|
47
70
|
end
|
|
48
71
|
|
|
72
|
+
def execute_request(env)
|
|
73
|
+
status, headers, body = @app.call(env)
|
|
74
|
+
Context.current&.set(:http_status, status)
|
|
75
|
+
enrich_user_context(env)
|
|
76
|
+
[status, headers, body]
|
|
77
|
+
end
|
|
78
|
+
|
|
49
79
|
def enrich_user_context(env)
|
|
50
80
|
event = Context.current
|
|
51
81
|
return unless event
|
|
@@ -72,41 +102,16 @@ module CanonicalLog
|
|
|
72
102
|
nil
|
|
73
103
|
end
|
|
74
104
|
|
|
105
|
+
def finalize!
|
|
106
|
+
emit! if Context.current
|
|
107
|
+
Context.clear!
|
|
108
|
+
end
|
|
109
|
+
|
|
75
110
|
def emit!
|
|
76
111
|
event = Context.current
|
|
77
112
|
return unless event
|
|
78
113
|
|
|
79
|
-
|
|
80
|
-
config.before_emit&.call(event)
|
|
81
|
-
|
|
82
|
-
event_hash = event.to_h
|
|
83
|
-
return unless config.should_sample?(event_hash)
|
|
84
|
-
|
|
85
|
-
event_hash[:message] ||= build_message(event_hash)
|
|
86
|
-
|
|
87
|
-
json = event_hash.to_json
|
|
88
|
-
|
|
89
|
-
config.resolved_sinks.each do |sink|
|
|
90
|
-
sink.write(json)
|
|
91
|
-
rescue StandardError => e
|
|
92
|
-
warn "[CanonicalLog] Sink error (#{sink.class}): #{e.message}"
|
|
93
|
-
end
|
|
94
|
-
rescue StandardError => e
|
|
95
|
-
warn "[CanonicalLog] Emit error: #{e.message}"
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
def build_message(event_hash)
|
|
99
|
-
[event_hash[:http_method], event_hash[:path], event_hash[:http_status]].compact.join(' ')
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
def ignored_path?(env)
|
|
103
|
-
path = env['PATH_INFO']
|
|
104
|
-
CanonicalLog.configuration.ignored_paths.any? do |pattern|
|
|
105
|
-
case pattern
|
|
106
|
-
when Regexp then pattern.match?(path)
|
|
107
|
-
when String then path.start_with?(pattern)
|
|
108
|
-
end
|
|
109
|
-
end
|
|
114
|
+
Emitter.emit!(event)
|
|
110
115
|
end
|
|
111
116
|
end
|
|
112
117
|
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CanonicalLog
|
|
4
|
+
module RailsLogSuppressor
|
|
5
|
+
SUPPRESSED_SUBSCRIBERS = [
|
|
6
|
+
'ActionController::LogSubscriber',
|
|
7
|
+
'ActionView::LogSubscriber',
|
|
8
|
+
'ActiveRecord::LogSubscriber',
|
|
9
|
+
].freeze
|
|
10
|
+
|
|
11
|
+
def self.suppress!
|
|
12
|
+
suppress_log_subscribers!
|
|
13
|
+
suppress_rack_logger!
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def self.suppress_log_subscribers!
|
|
17
|
+
null_logger = Logger.new(File::NULL)
|
|
18
|
+
|
|
19
|
+
ActiveSupport::LogSubscriber.log_subscribers.each do |subscriber|
|
|
20
|
+
next unless SUPPRESSED_SUBSCRIBERS.include?(subscriber.class.name)
|
|
21
|
+
|
|
22
|
+
subscriber.logger = null_logger
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def self.suppress_rack_logger!
|
|
27
|
+
return unless defined?(Rails::Rack::Logger)
|
|
28
|
+
|
|
29
|
+
Rails::Rack::Logger.prepend(SilentRackLogger)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Keeps Rails::Rack::Logger in the middleware stack (preserves tagged logging
|
|
33
|
+
# and request_id setup) but silences its log output.
|
|
34
|
+
module SilentRackLogger
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def started_request_message(_request)
|
|
38
|
+
nil
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def logger
|
|
42
|
+
@logger ||= Logger.new(File::NULL)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
module CanonicalLog
|
|
4
4
|
class Railtie < Rails::Railtie
|
|
5
|
+
config.after_initialize do
|
|
6
|
+
CanonicalLog::RailsLogSuppressor.suppress! if CanonicalLog.configuration.suppress_rails_logging
|
|
7
|
+
end
|
|
8
|
+
|
|
5
9
|
initializer 'canonical_log.insert_middleware' do |app|
|
|
6
10
|
app.middleware.insert(0, CanonicalLog::Middleware)
|
|
7
11
|
end
|
|
@@ -9,6 +13,8 @@ module CanonicalLog
|
|
|
9
13
|
initializer 'canonical_log.subscribe' do
|
|
10
14
|
CanonicalLog::Subscribers::ActionController.subscribe!
|
|
11
15
|
CanonicalLog::Subscribers::ActiveRecord.subscribe!
|
|
16
|
+
CanonicalLog::Subscribers::ActiveSupportCache.subscribe!
|
|
17
|
+
CanonicalLog::Subscribers::ActiveJob.subscribe! if defined?(ActiveJob)
|
|
12
18
|
end
|
|
13
19
|
end
|
|
14
20
|
end
|
|
@@ -7,14 +7,10 @@ module CanonicalLog
|
|
|
7
7
|
status = event_hash[:http_status] || 0
|
|
8
8
|
duration = event_hash[:duration_ms] || 0
|
|
9
9
|
|
|
10
|
-
# Always keep errors
|
|
11
|
-
return true if status >= 500
|
|
12
|
-
return true if event_hash[:error]
|
|
13
|
-
|
|
14
|
-
# Always keep slow requests
|
|
10
|
+
# Always keep errors and slow requests
|
|
11
|
+
return true if status >= 500 || event_hash[:error]
|
|
15
12
|
return true if duration >= config.slow_request_threshold_ms
|
|
16
13
|
|
|
17
|
-
# Sample the rest
|
|
18
14
|
rand < config.sample_rate
|
|
19
15
|
end
|
|
20
16
|
end
|
|
@@ -5,7 +5,7 @@ module CanonicalLog
|
|
|
5
5
|
# Duck-type interface for sinks.
|
|
6
6
|
# Any object responding to #write(json_string) can be used as a sink.
|
|
7
7
|
class Base
|
|
8
|
-
def write(json_string)
|
|
8
|
+
def write(json_string, level: :info)
|
|
9
9
|
raise NotImplementedError, "#{self.class} must implement #write"
|
|
10
10
|
end
|
|
11
11
|
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CanonicalLog
|
|
4
|
+
module SqlSanitizer
|
|
5
|
+
# Matches single-quoted string literals (including escaped quotes)
|
|
6
|
+
STRING_LITERAL = /'(?:[^'\\]|\\.)*'/
|
|
7
|
+
|
|
8
|
+
# Matches numeric literals in value positions (after =, >, <, IN (, comma, etc.)
|
|
9
|
+
NUMERIC_LITERAL = /(?<=[\s=><,(])-?\b\d+(?:\.\d+)?\b/
|
|
10
|
+
|
|
11
|
+
def self.sanitize(sql)
|
|
12
|
+
return sql unless sql.is_a?(String)
|
|
13
|
+
|
|
14
|
+
result = sql.gsub(STRING_LITERAL, "'?'")
|
|
15
|
+
result.gsub(NUMERIC_LITERAL, '?')
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -11,31 +11,45 @@ module CanonicalLog
|
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def self.handle(notification)
|
|
14
|
+
return unless CanonicalLog.configuration.enabled
|
|
15
|
+
|
|
14
16
|
event = Context.current
|
|
15
17
|
return unless event
|
|
16
18
|
|
|
17
19
|
payload = notification.payload
|
|
18
|
-
|
|
20
|
+
event.add(extract_fields(payload))
|
|
21
|
+
end
|
|
19
22
|
|
|
23
|
+
def self.extract_fields(payload)
|
|
20
24
|
params = (payload[:params] || {}).except('controller', 'action')
|
|
21
|
-
filtered_params = filter_params(params,
|
|
25
|
+
filtered_params = filter_params(params, CanonicalLog.configuration.param_filter_keys)
|
|
22
26
|
|
|
23
|
-
|
|
27
|
+
{
|
|
24
28
|
controller: payload[:controller],
|
|
25
29
|
action: payload[:action],
|
|
26
30
|
format: payload[:format],
|
|
27
31
|
params: filtered_params,
|
|
28
32
|
view_runtime_ms: payload[:view_runtime]&.round(2),
|
|
29
|
-
db_runtime_ms: payload[:db_runtime]&.round(2)
|
|
30
|
-
|
|
33
|
+
db_runtime_ms: payload[:db_runtime]&.round(2),
|
|
34
|
+
}
|
|
31
35
|
end
|
|
32
36
|
|
|
33
37
|
def self.filter_params(params, filter_keys)
|
|
38
|
+
if defined?(ActiveSupport::ParameterFilter)
|
|
39
|
+
ActiveSupport::ParameterFilter.new(filter_keys).filter(params)
|
|
40
|
+
else
|
|
41
|
+
deep_filter(params, filter_keys)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.deep_filter(params, filter_keys)
|
|
34
46
|
params.each_with_object({}) do |(key, value), filtered|
|
|
35
47
|
filtered[key] = if filter_keys.include?(key.to_s)
|
|
36
48
|
'[FILTERED]'
|
|
37
49
|
elsif value.is_a?(Hash)
|
|
38
|
-
|
|
50
|
+
deep_filter(value, filter_keys)
|
|
51
|
+
elsif value.is_a?(Array)
|
|
52
|
+
value.map { |v| v.is_a?(Hash) ? deep_filter(v, filter_keys) : v }
|
|
39
53
|
else
|
|
40
54
|
value
|
|
41
55
|
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CanonicalLog
|
|
4
|
+
module Subscribers
|
|
5
|
+
module ActiveJob
|
|
6
|
+
def self.subscribe!
|
|
7
|
+
ActiveSupport::Notifications.subscribe('perform.active_job') do |*args|
|
|
8
|
+
notification = ActiveSupport::Notifications::Event.new(*args)
|
|
9
|
+
handle(notification)
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def self.handle(notification)
|
|
14
|
+
return unless CanonicalLog.configuration.enabled
|
|
15
|
+
|
|
16
|
+
Context.init!
|
|
17
|
+
event = Context.current
|
|
18
|
+
enrich_job_fields(event, notification.payload)
|
|
19
|
+
event.set(:duration_ms, notification.duration.round(2))
|
|
20
|
+
Emitter.emit!(event)
|
|
21
|
+
ensure
|
|
22
|
+
Context.clear!
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.enrich_job_fields(event, payload)
|
|
26
|
+
job = payload[:job]
|
|
27
|
+
event.add(
|
|
28
|
+
job_class: job.class.name,
|
|
29
|
+
queue: job.queue_name,
|
|
30
|
+
job_id: job.job_id,
|
|
31
|
+
executions: job.executions,
|
|
32
|
+
priority: job.priority,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
return unless payload[:exception_object]
|
|
36
|
+
|
|
37
|
+
event.add(
|
|
38
|
+
error_class: payload[:exception_object].class.name,
|
|
39
|
+
error_message: payload[:exception_object].message,
|
|
40
|
+
)
|
|
41
|
+
end
|
|
42
|
+
private_class_method :enrich_job_fields
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -11,24 +11,35 @@ module CanonicalLog
|
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def self.handle(notification)
|
|
14
|
+
return unless CanonicalLog.configuration.enabled
|
|
14
15
|
event = Context.current
|
|
15
16
|
return unless event
|
|
16
|
-
|
|
17
17
|
payload = notification.payload
|
|
18
|
-
return if
|
|
18
|
+
return if ['SCHEMA', 'CACHE'].include?(payload[:name])
|
|
19
|
+
|
|
20
|
+
track_query_metrics(event, notification.duration)
|
|
21
|
+
capture_slow_query(event, notification.duration, payload)
|
|
22
|
+
end
|
|
19
23
|
|
|
20
|
-
|
|
24
|
+
def self.track_query_metrics(event, duration_ms)
|
|
21
25
|
event.increment(:db_query_count)
|
|
22
26
|
event.increment(:db_total_time_ms, duration_ms.round(2))
|
|
27
|
+
end
|
|
23
28
|
|
|
29
|
+
def self.capture_slow_query(event, duration_ms, payload)
|
|
24
30
|
threshold = CanonicalLog.configuration.slow_query_threshold_ms
|
|
25
31
|
return unless duration_ms >= threshold
|
|
26
32
|
|
|
33
|
+
sql = resolve_sql(payload[:sql])
|
|
27
34
|
event.append(:slow_queries, {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
35
|
+
sql: sql,
|
|
36
|
+
duration_ms: duration_ms.round(2),
|
|
37
|
+
name: payload[:name],
|
|
38
|
+
})
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def self.resolve_sql(raw_sql)
|
|
42
|
+
CanonicalLog.configuration.filter_sql_literals ? SqlSanitizer.sanitize(raw_sql) : raw_sql
|
|
32
43
|
end
|
|
33
44
|
end
|
|
34
45
|
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CanonicalLog
|
|
4
|
+
module Subscribers
|
|
5
|
+
module ActiveSupportCache
|
|
6
|
+
EVENTS = [
|
|
7
|
+
'cache_read.active_support',
|
|
8
|
+
'cache_write.active_support',
|
|
9
|
+
'cache_fetch_hit.active_support',
|
|
10
|
+
].freeze
|
|
11
|
+
|
|
12
|
+
def self.subscribe!
|
|
13
|
+
EVENTS.each do |event_name|
|
|
14
|
+
ActiveSupport::Notifications.subscribe(event_name) do |*args|
|
|
15
|
+
event = ActiveSupport::Notifications::Event.new(*args)
|
|
16
|
+
handle(event)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.handle(notification)
|
|
22
|
+
return unless CanonicalLog.configuration.enabled
|
|
23
|
+
|
|
24
|
+
event = Context.current
|
|
25
|
+
return unless event
|
|
26
|
+
|
|
27
|
+
event.increment(:cache_total_time_ms, notification.duration.round(2))
|
|
28
|
+
track_operation(event, notification)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def self.track_operation(event, notification)
|
|
32
|
+
case notification.name
|
|
33
|
+
when 'cache_read.active_support'
|
|
34
|
+
event.increment(:cache_read_count)
|
|
35
|
+
event.increment(notification.payload[:hit] ? :cache_hit_count : :cache_miss_count)
|
|
36
|
+
when 'cache_write.active_support'
|
|
37
|
+
event.increment(:cache_write_count)
|
|
38
|
+
when 'cache_fetch_hit.active_support'
|
|
39
|
+
event.increment(:cache_read_count)
|
|
40
|
+
event.increment(:cache_hit_count)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
private_class_method :track_operation
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
data/lib/canonical_log.rb
CHANGED
|
@@ -7,13 +7,21 @@ require_relative 'canonical_log/configuration'
|
|
|
7
7
|
require_relative 'canonical_log/sampling'
|
|
8
8
|
require_relative 'canonical_log/event'
|
|
9
9
|
require_relative 'canonical_log/context'
|
|
10
|
+
require_relative 'canonical_log/emitter'
|
|
11
|
+
require_relative 'canonical_log/sql_sanitizer'
|
|
10
12
|
require_relative 'canonical_log/middleware'
|
|
11
13
|
require_relative 'canonical_log/sinks/base'
|
|
12
14
|
require_relative 'canonical_log/sinks/stdout'
|
|
15
|
+
require_relative 'canonical_log/sinks/null'
|
|
13
16
|
require_relative 'canonical_log/sinks/rails_logger'
|
|
14
17
|
require_relative 'canonical_log/subscribers/action_controller'
|
|
15
18
|
require_relative 'canonical_log/subscribers/active_record'
|
|
19
|
+
require_relative 'canonical_log/subscribers/active_support_cache'
|
|
20
|
+
require_relative 'canonical_log/formatters/pretty'
|
|
21
|
+
require_relative 'canonical_log/formatters/logfmt'
|
|
22
|
+
require_relative 'canonical_log/rails_log_suppressor'
|
|
16
23
|
require_relative 'canonical_log/integrations/error_enrichment'
|
|
24
|
+
require_relative 'canonical_log/subscribers/active_job' if defined?(ActiveJob)
|
|
17
25
|
require_relative 'canonical_log/integrations/sidekiq' if defined?(Sidekiq)
|
|
18
26
|
|
|
19
27
|
require_relative 'canonical_log/railtie' if defined?(Rails::Railtie)
|
|
@@ -1,35 +1,62 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
CanonicalLog.configure do |config|
|
|
4
|
-
#
|
|
5
|
-
|
|
4
|
+
# Master on/off switch. Defaults to true in production, false otherwise.
|
|
5
|
+
config.enabled = Rails.env.production?
|
|
6
|
+
|
|
7
|
+
# Silence Rails' built-in ActionController/ActionView log output.
|
|
8
|
+
config.suppress_rails_logging = false
|
|
9
|
+
|
|
10
|
+
# Shortcut: enable colorized, indented JSON output (sets format to :pretty).
|
|
11
|
+
config.pretty = Rails.env.development?
|
|
12
|
+
|
|
13
|
+
# Output format: :json (default), :pretty (colorized JSON), :logfmt (key=value).
|
|
14
|
+
# config.format = :json
|
|
15
|
+
|
|
16
|
+
# Where to write log lines. :auto sends JSON to $stdout.
|
|
6
17
|
# config.sinks = :auto
|
|
7
|
-
# config.sinks = [CanonicalLog::Sinks::Stdout.new]
|
|
8
|
-
# config.sinks = [CanonicalLog::Sinks::RailsLogger.new]
|
|
9
18
|
|
|
10
|
-
# Parameter keys
|
|
19
|
+
# Parameter keys replaced with [FILTERED] in params and query strings.
|
|
11
20
|
# config.param_filter_keys = %w[password password_confirmation token secret]
|
|
12
21
|
|
|
13
|
-
#
|
|
22
|
+
# Replace literal values in SQL captured as slow queries.
|
|
23
|
+
# config.filter_sql_literals = true
|
|
24
|
+
|
|
25
|
+
# Filter sensitive params from the query_string field.
|
|
26
|
+
# config.filter_query_string = true
|
|
27
|
+
|
|
28
|
+
# SQL queries slower than this (ms) are captured in slow_queries.
|
|
14
29
|
# config.slow_query_threshold_ms = 100.0
|
|
15
30
|
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
|
|
31
|
+
# Requests slower than this (ms) are always logged, even when sampled out.
|
|
32
|
+
# config.slow_request_threshold_ms = 2000.0
|
|
33
|
+
|
|
34
|
+
# Fraction of requests to log (1.0 = all). Errors and slow requests are always kept.
|
|
35
|
+
# config.sample_rate = 1.0
|
|
36
|
+
|
|
37
|
+
# Custom sampling: ->(event_hash, config) { true/false }. Overrides sample_rate.
|
|
38
|
+
# config.sampling = nil
|
|
39
|
+
|
|
40
|
+
# Number of backtrace lines in structured errors (0 to disable).
|
|
41
|
+
# config.error_backtrace_lines = 5
|
|
42
|
+
|
|
43
|
+
# Custom log level: ->(event_hash) { :info/:warn/:error }. Default: 5xx->error, 4xx->warn.
|
|
44
|
+
# config.log_level_resolver = nil
|
|
45
|
+
|
|
46
|
+
# Static fields merged into every event.
|
|
47
|
+
# config.default_fields = {}
|
|
48
|
+
|
|
49
|
+
# Extract user context from Rack env. Without this, Warden/Devise is auto-detected.
|
|
50
|
+
# config.user_context = ->(env) {
|
|
51
|
+
# user = env["warden"]&.user
|
|
52
|
+
# user ? { user_id: user.id } : {}
|
|
25
53
|
# }
|
|
26
54
|
|
|
27
|
-
# Hook called with the Event just before
|
|
55
|
+
# Hook called with the Event just before emission.
|
|
28
56
|
# config.before_emit = ->(event) {
|
|
29
57
|
# event.set(:app_version, ENV["APP_VERSION"])
|
|
30
58
|
# }
|
|
31
59
|
|
|
32
|
-
# Paths to
|
|
33
|
-
# Supports strings (prefix match) and regexps.
|
|
60
|
+
# Paths to skip entirely. Strings match by prefix, Regexps by pattern.
|
|
34
61
|
# config.ignored_paths = ["/health", "/assets", %r{\A/packs}]
|
|
35
62
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: canonical_log
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Krzysztof Duda
|
|
@@ -50,17 +50,25 @@ files:
|
|
|
50
50
|
- lib/canonical_log.rb
|
|
51
51
|
- lib/canonical_log/configuration.rb
|
|
52
52
|
- lib/canonical_log/context.rb
|
|
53
|
+
- lib/canonical_log/emitter.rb
|
|
53
54
|
- lib/canonical_log/event.rb
|
|
55
|
+
- lib/canonical_log/formatters/logfmt.rb
|
|
56
|
+
- lib/canonical_log/formatters/pretty.rb
|
|
54
57
|
- lib/canonical_log/integrations/error_enrichment.rb
|
|
55
58
|
- lib/canonical_log/integrations/sidekiq.rb
|
|
56
59
|
- lib/canonical_log/middleware.rb
|
|
60
|
+
- lib/canonical_log/rails_log_suppressor.rb
|
|
57
61
|
- lib/canonical_log/railtie.rb
|
|
58
62
|
- lib/canonical_log/sampling.rb
|
|
59
63
|
- lib/canonical_log/sinks/base.rb
|
|
64
|
+
- lib/canonical_log/sinks/null.rb
|
|
60
65
|
- lib/canonical_log/sinks/rails_logger.rb
|
|
61
66
|
- lib/canonical_log/sinks/stdout.rb
|
|
67
|
+
- lib/canonical_log/sql_sanitizer.rb
|
|
62
68
|
- lib/canonical_log/subscribers/action_controller.rb
|
|
69
|
+
- lib/canonical_log/subscribers/active_job.rb
|
|
63
70
|
- lib/canonical_log/subscribers/active_record.rb
|
|
71
|
+
- lib/canonical_log/subscribers/active_support_cache.rb
|
|
64
72
|
- lib/canonical_log/version.rb
|
|
65
73
|
- lib/generators/canonical_log/install_generator.rb
|
|
66
74
|
- lib/generators/canonical_log/templates/canonical_log.rb
|