logstruct 0.1.12 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +139 -1
  3. data/README.md +76 -68
  4. data/lib/log_struct/concerns/configuration.rb +139 -91
  5. data/lib/log_struct/concerns/error_handling.rb +36 -6
  6. data/lib/log_struct/config_struct/error_handling_modes.rb +3 -1
  7. data/lib/log_struct/config_struct/filters.rb +116 -33
  8. data/lib/log_struct/config_struct/integrations.rb +0 -4
  9. data/lib/log_struct/config_struct/limits.rb +29 -0
  10. data/lib/log_struct/configuration.rb +2 -0
  11. data/lib/log_struct/enums/event.rb +3 -1
  12. data/lib/log_struct/enums/log_field.rb +5 -0
  13. data/lib/log_struct/formatter.rb +304 -56
  14. data/lib/log_struct/hash_utils.rb +33 -4
  15. data/lib/log_struct/integrations/action_mailer/error_handling.rb +78 -20
  16. data/lib/log_struct/integrations/action_mailer/fallback_error_handlers.rb +81 -0
  17. data/lib/log_struct/integrations/action_mailer/metadata_collection.rb +0 -20
  18. data/lib/log_struct/integrations/action_mailer.rb +22 -17
  19. data/lib/log_struct/integrations/active_job/log_subscriber.rb +177 -25
  20. data/lib/log_struct/integrations/active_job.rb +2 -7
  21. data/lib/log_struct/integrations/active_model_serializers.rb +7 -1
  22. data/lib/log_struct/integrations/active_record.rb +39 -17
  23. data/lib/log_struct/integrations/active_storage.rb +31 -31
  24. data/lib/log_struct/integrations/ahoy.rb +19 -15
  25. data/lib/log_struct/integrations/carrierwave.rb +55 -51
  26. data/lib/log_struct/integrations/dotenv.rb +22 -32
  27. data/lib/log_struct/integrations/good_job/logger.rb +30 -24
  28. data/lib/log_struct/integrations/good_job.rb +111 -40
  29. data/lib/log_struct/integrations/host_authorization.rb +47 -36
  30. data/lib/log_struct/integrations/lograge.rb +80 -4
  31. data/lib/log_struct/integrations/puma.rb +74 -34
  32. data/lib/log_struct/integrations/rack_error_handler/debug_exceptions_logging.rb +35 -0
  33. data/lib/log_struct/integrations/rack_error_handler/error_lines.rb +160 -0
  34. data/lib/log_struct/integrations/rack_error_handler/middleware.rb +14 -169
  35. data/lib/log_struct/integrations/rack_error_handler/parse_error_logging.rb +36 -0
  36. data/lib/log_struct/integrations/rack_error_handler.rb +20 -4
  37. data/lib/log_struct/integrations/rack_middleware.rb +23 -0
  38. data/lib/log_struct/integrations/rails_log_subscribers.rb +32 -0
  39. data/lib/log_struct/integrations/request_context/middleware.rb +4 -9
  40. data/lib/log_struct/integrations/shrine.rb +48 -73
  41. data/lib/log_struct/integrations/sidekiq/job_context_middleware.rb +32 -0
  42. data/lib/log_struct/integrations/sidekiq/logger.rb +17 -9
  43. data/lib/log_struct/integrations/sidekiq.rb +98 -7
  44. data/lib/log_struct/integrations/sorbet.rb +10 -46
  45. data/lib/log_struct/integrations.rb +3 -2
  46. data/lib/log_struct/log/{good_job/enqueue.rb → active_job/discard.rb} +13 -13
  47. data/lib/log_struct/log/active_job/enqueue.rb +0 -4
  48. data/lib/log_struct/log/{good_job/schedule.rb → active_job/error.rb} +15 -15
  49. data/lib/log_struct/log/{good_job/start.rb → active_job/retry.rb} +13 -13
  50. data/lib/log_struct/log/{good_job/finish.rb → active_job/retry_stopped.rb} +13 -17
  51. data/lib/log_struct/log/active_job.rb +4 -0
  52. data/lib/log_struct/log/active_storage/delete.rb +6 -2
  53. data/lib/log_struct/log/active_storage/download.rb +2 -4
  54. data/lib/log_struct/log/active_storage/exist.rb +2 -2
  55. data/lib/log_struct/log/active_storage/metadata.rb +4 -4
  56. data/lib/log_struct/log/active_storage/upload.rb +2 -10
  57. data/lib/log_struct/log/active_storage/url.rb +2 -6
  58. data/lib/log_struct/log/active_storage.rb +2 -3
  59. data/lib/log_struct/log/error.rb +24 -1
  60. data/lib/log_struct/log/good_job.rb +0 -4
  61. data/lib/log_struct/log/puma/error.rb +63 -0
  62. data/lib/log_struct/log/puma.rb +1 -0
  63. data/lib/log_struct/log/security/blocked_host.rb +0 -4
  64. data/lib/log_struct/log/shrine/exist.rb +0 -2
  65. data/lib/log_struct/log/shrine/metadata.rb +2 -4
  66. data/lib/log_struct/log/shrine/upload.rb +4 -0
  67. data/lib/log_struct/log/sidekiq/error.rb +63 -0
  68. data/lib/log_struct/log/{active_storage/stream.rb → sidekiq/log.rb} +17 -11
  69. data/lib/log_struct/log/sidekiq.rb +3 -45
  70. data/lib/log_struct/log.rb +4 -2
  71. data/lib/log_struct/logged_errors.rb +25 -0
  72. data/lib/log_struct/monkey_patches/active_support/tagged_logging/formatter.rb +2 -3
  73. data/lib/log_struct/param_filters.rb +51 -38
  74. data/lib/log_struct/params_errors.rb +78 -0
  75. data/lib/log_struct/rails_boot_banner_silencer.rb +7 -2
  76. data/lib/log_struct/railtie.rb +29 -14
  77. data/lib/log_struct/semantic_logger/color_formatter.rb +60 -53
  78. data/lib/log_struct/semantic_logger/concerns/log_methods.rb +65 -57
  79. data/lib/log_struct/semantic_logger/formatter.rb +79 -57
  80. data/lib/log_struct/semantic_logger/logger.rb +29 -53
  81. data/lib/log_struct/semantic_logger/reopen_after_fork.rb +40 -0
  82. data/lib/log_struct/semantic_logger/setup.rb +115 -112
  83. data/lib/log_struct/shared/serialize_common.rb +2 -7
  84. data/lib/log_struct/string_scrubber.rb +229 -20
  85. data/lib/log_struct/version.rb +1 -1
  86. data/lib/log_struct.rb +6 -4
  87. data/logstruct.gemspec +19 -10
  88. metadata +63 -11
  89. data/lib/log_struct/builders/active_job.rb +0 -87
  90. data/lib/log_struct/integrations/good_job/log_subscriber.rb +0 -189
  91. data/lib/log_struct/integrations/rack.rb +0 -31
@@ -8,29 +8,17 @@ rescue LoadError
8
8
  end
9
9
 
10
10
  require_relative "good_job/logger" if defined?(::GoodJob)
11
- require_relative "good_job/log_subscriber" if defined?(::GoodJob)
12
11
 
13
12
  module LogStruct
14
13
  module Integrations
15
14
  # GoodJob integration for structured logging
16
15
  #
17
- # GoodJob is a PostgreSQL-based ActiveJob backend that provides reliable,
18
- # scalable job processing for Rails applications. This integration provides
19
- # structured logging for all GoodJob operations.
16
+ # GoodJob is a PostgreSQL-based ActiveJob backend. Its jobs are logged by the ActiveJob
17
+ # integration (enqueued, started, finished, failed), so this integration covers GoodJob itself:
20
18
  #
21
- # ## Features:
22
- # - Structured logging for job execution lifecycle
23
- # - Error tracking and retry logging
24
- # - Performance metrics and timing data
25
- # - Database operation logging
26
- # - Thread and process tracking
27
- # - Custom GoodJob logger with LogStruct formatting
28
- #
29
- # ## Integration Points:
30
- # - Replaces GoodJob.logger with LogStruct-compatible logger
31
- # - Subscribes to GoodJob's ActiveSupport notifications
32
- # - Captures job execution events, errors, and performance metrics
33
- # - Logs database operations and connection information
19
+ # - Replaces GoodJob.logger with a LogStruct logger, so GoodJob's own messages are JSON lines
20
+ # - Logs each thread error as a structured error line, then calls the app's `on_thread_error`
21
+ # - Stops GoodJob's own log subscriber logging those errors again with the whole backtrace
34
22
  #
35
23
  # ## Configuration:
36
24
  # The integration is automatically enabled when GoodJob is detected and
@@ -45,8 +33,7 @@ module LogStruct
45
33
 
46
34
  # Set up GoodJob structured logging
47
35
  #
48
- # This method configures GoodJob to use LogStruct's structured logging
49
- # by replacing the default logger and subscribing to job events.
36
+ # This method replaces GoodJob's logger and logs its thread errors as structured lines.
50
37
  #
51
38
  # @param config [LogStruct::Configuration] The LogStruct configuration
52
39
  # @return [Boolean, nil] Returns true if setup was successful, nil if skipped
@@ -59,51 +46,135 @@ module LogStruct
59
46
  # Replace GoodJob's logger with our structured logger
60
47
  configure_logger
61
48
 
62
- # Subscribe to GoodJob's ActiveSupport notifications
63
- subscribe_to_notifications
49
+ compact_error_lines
64
50
 
65
51
  true
66
52
  end
67
53
 
54
+ # GoodJob::LogSubscriber logs errors as text with the whole backtrace in the message (over 10KB).
55
+ # GoodJob passes every error its job tasks raise, every systemd watchdog error and every notifier
56
+ # error except connection errors to `on_thread_error`, which LogThreadErrors logs as a structured
57
+ # line, so GoodJob's own lines for those are off. The notifier passes connection errors on only
58
+ # once they repeat, so its line for them is kept, without the backtrace.
59
+ module CompactErrorLines
60
+ extend T::Sig
61
+ extend T::Helpers
62
+
63
+ requires_ancestor { ::GoodJob::LogSubscriber }
64
+
65
+ sig { params(_event: T.untyped).void }
66
+ def finished_job_task(_event)
67
+ end
68
+
69
+ sig { params(event: T.untyped).void }
70
+ def notifier_notify_error(event)
71
+ exception = event.payload[:error]
72
+ return unless exception
73
+ return unless LogStruct::Integrations::GoodJob.notifier_connection_error?(exception)
74
+
75
+ error { "Notifier errored: #{exception.class}: #{exception}" }
76
+ end
77
+
78
+ sig { params(_event: T.untyped).void }
79
+ def systemd_watchdog_error(_event)
80
+ end
81
+ end
82
+
83
+ # Logs each thread error as a structured line, then calls the handler the app set as
84
+ # `on_thread_error` (GoodJob's README sets one to report errors to an error tracker). Every
85
+ # GoodJob component passes its thread errors to `GoodJob._on_thread_error`, so this works
86
+ # whenever the app sets its handler: GoodJob's engine applies config.good_job.on_thread_error
87
+ # after LogStruct's setup. GoodJob passes any Exception its handled_exceptions setting lets
88
+ # through, not only StandardErrors.
89
+ module LogThreadErrors
90
+ extend T::Sig
91
+
92
+ sig { params(exception: Exception).void }
93
+ def _on_thread_error(exception)
94
+ LogStruct::Integrations::GoodJob.log_thread_error(exception)
95
+ rescue => e
96
+ LogStruct.handle_exception(e, source: LogStruct::Source::Internal, context: {integration: :good_job})
97
+ ensure
98
+ # The app's handler runs even if LogStruct's line failed
99
+ super
100
+ end
101
+ end
102
+
68
103
  # Configure GoodJob to use LogStruct's structured logger
69
104
  sig { void }
70
105
  def self.configure_logger
71
106
  return unless defined?(::GoodJob)
72
107
 
73
- # Use direct reference to avoid const_get - GoodJob is guaranteed to be defined here
74
- goodjob_module = T.unsafe(GoodJob)
108
+ # Not GoodJob, which here is this module
109
+ goodjob_module = ::GoodJob
75
110
 
76
111
  # Replace GoodJob.logger with our structured logger if GoodJob is available
77
112
  if goodjob_module.respond_to?(:logger=)
78
113
  goodjob_module.logger = LogStruct::Integrations::GoodJob::Logger.new("GoodJob")
79
- end
80
114
 
81
- # Configure error handling for thread errors if GoodJob supports it
82
- if goodjob_module.respond_to?(:on_thread_error=)
83
- goodjob_module.on_thread_error = ->(exception) do
84
- log_entry = LogStruct::Log::GoodJob::Error.new(
85
- error_class: exception.class.name,
86
- error_message: exception.message,
87
- backtrace: exception.backtrace,
88
- process_id: ::Process.pid,
89
- thread_id: Thread.current.object_id.to_s(36)
90
- )
91
- goodjob_module.logger.error(log_entry)
115
+ # GoodJob's executable adds a plain text STDOUT logger unless GoodJob.logger already writes
116
+ # to STDOUT, which it can't tell for this logger
117
+ if defined?(::GoodJob::CLI) && ::GoodJob::CLI.respond_to?(:log_to_stdout=)
118
+ ::GoodJob::CLI.log_to_stdout = false
92
119
  end
93
120
  end
121
+
122
+ configure_thread_error_handler
94
123
  end
95
124
 
96
- # Subscribe to GoodJob's ActiveSupport notifications
97
125
  sig { void }
98
- def self.subscribe_to_notifications
126
+ def self.configure_thread_error_handler
99
127
  return unless defined?(::GoodJob)
100
128
 
101
- # Subscribe to our custom log subscriber for GoodJob events
102
- LogStruct::Integrations::GoodJob::LogSubscriber.attach_to :good_job
129
+ # Not GoodJob, which here is this module
130
+ goodjob_singleton = ::GoodJob.singleton_class
131
+ return unless goodjob_singleton.method_defined?(:_on_thread_error)
132
+
133
+ goodjob_singleton.prepend(LogThreadErrors) unless goodjob_singleton.ancestors.include?(LogThreadErrors)
134
+ end
135
+
136
+ # A failed job's exception was already logged by the ActiveJob integration, with the job's
137
+ # ID and class, so it isn't logged again here
138
+ sig { params(exception: Exception).void }
139
+ def self.log_thread_error(exception)
140
+ return if LogStruct::LoggedErrors.logged?(exception)
141
+
142
+ ::GoodJob.logger.error(
143
+ LogStruct::Log::GoodJob::Error.new(
144
+ error_class: exception.class.name || exception.class.to_s,
145
+ error_message: exception.message,
146
+ backtrace: exception.backtrace,
147
+ level: LogStruct::Level::Error,
148
+ process_id: ::Process.pid,
149
+ thread_id: Thread.current.object_id.to_s(36)
150
+ )
151
+ )
152
+ end
153
+
154
+ # The errors GoodJob's notifier holds back until they repeat (GoodJob::Notifier::CONNECTION_ERRORS)
155
+ sig { params(exception: Exception).returns(T::Boolean) }
156
+ def self.notifier_connection_error?(exception)
157
+ return true unless defined?(::GoodJob::Notifier::CONNECTION_ERRORS)
158
+
159
+ ::GoodJob::Notifier::CONNECTION_ERRORS.any? do |error_name|
160
+ error_class = error_name.safe_constantize
161
+ error_class.is_a?(Class) && exception.is_a?(error_class)
162
+ end
163
+ end
164
+
165
+ # Only when LogThreadErrors logs thread errors instead
166
+ sig { void }
167
+ def self.compact_error_lines
168
+ return unless defined?(::GoodJob::LogSubscriber)
169
+ return unless ::GoodJob.singleton_class.ancestors.include?(LogThreadErrors)
170
+
171
+ log_subscriber = ::GoodJob::LogSubscriber
172
+ log_subscriber.prepend(CompactErrorLines) unless log_subscriber.ancestors.include?(CompactErrorLines)
103
173
  end
104
174
 
105
175
  private_class_method :configure_logger
106
- private_class_method :subscribe_to_notifications
176
+ private_class_method :compact_error_lines
177
+ private_class_method :configure_thread_error_handler
107
178
  end
108
179
  end
109
180
  end
@@ -35,57 +35,68 @@ module LogStruct
35
35
  return nil unless config.enabled
36
36
  return nil unless config.integrations.enable_host_authorization
37
37
 
38
- # Define the response app as a separate variable to fix block alignment
39
- response_app = lambda do |env|
40
- request = ::ActionDispatch::Request.new(env)
41
- # Include the blocked hosts app configuration in the log entry
42
- # This can be helpful later when reviewing logs.
43
- blocked_hosts = env["action_dispatch.blocked_hosts"]
38
+ # Merge our response_app into existing host_authorization config to preserve excludes
39
+ existing = Rails.application.config.host_authorization
40
+ unless existing.is_a?(Hash)
41
+ existing = {}
42
+ end
43
+ existing = existing.dup
44
+ app_response_app = existing[:response_app]
45
+ # Set up once: don't wrap our own response app again
46
+ app_response_app = app_response_app.app_response_app if app_response_app.is_a?(ResponseApp)
47
+ existing[:response_app] = ResponseApp.new(app_response_app)
48
+ Rails.application.config.host_authorization = existing
44
49
 
45
- # Build allowed_hosts array
46
- allowed_hosts_array = T.let(nil, T.nilable(T::Array[String]))
47
- if blocked_hosts.respond_to?(:allowed_hosts)
48
- allowed_hosts_array = blocked_hosts.allowed_hosts
49
- end
50
+ true
51
+ end
52
+
53
+ # Logs the blocked host, then responds with the app's own response_app if it set one
54
+ class ResponseApp
55
+ extend T::Sig
50
56
 
51
- # Get allow_ip_hosts value
52
- allow_ip_hosts_value = T.let(nil, T.nilable(T::Boolean))
53
- if blocked_hosts.respond_to?(:allow_ip_hosts)
54
- allow_ip_hosts_value = blocked_hosts.allow_ip_hosts
57
+ sig { returns(T.untyped) }
58
+ attr_reader :app_response_app
59
+
60
+ sig { params(app_response_app: T.untyped).void }
61
+ def initialize(app_response_app)
62
+ @app_response_app = app_response_app
63
+ end
64
+
65
+ sig { params(env: T::Hash[String, T.untyped]).returns(T.untyped) }
66
+ def call(env)
67
+ begin
68
+ log_blocked_host(env)
69
+ rescue => e
70
+ LogStruct.handle_exception(e, source: LogStruct::Source::Internal, context: {integration: :host_authorization})
55
71
  end
72
+ return @app_response_app.call(env) if @app_response_app
73
+
74
+ # Dup the headers so they can be modified by downstream middleware
75
+ [FORBIDDEN_STATUS, RESPONSE_HEADERS.dup, [RESPONSE_HTML]]
76
+ end
77
+
78
+ private
79
+
80
+ sig { params(env: T::Hash[String, T.untyped]).void }
81
+ def log_blocked_host(env)
82
+ request = ::ActionDispatch::Request.new(env)
83
+ # The hosts Rails blocked: the Host header, and X-Forwarded-Host when it's set
84
+ blocked_hosts = Array(env["action_dispatch.blocked_hosts"]).map(&:to_s)
56
85
 
57
- # Create structured log entry for blocked host
58
86
  log_entry = LogStruct::Log::Security::BlockedHost.new(
59
87
  message: "Blocked host detected: #{request.host}",
60
88
  blocked_host: request.host,
89
+ blocked_hosts: blocked_hosts.empty? ? nil : blocked_hosts,
61
90
  path: request.path,
62
91
  http_method: request.method,
63
92
  source_ip: request.ip,
64
93
  user_agent: request.user_agent,
65
- referer: request.referer,
66
- x_forwarded_for: request.x_forwarded_for,
67
- allowed_hosts: allowed_hosts_array&.empty? ? nil : allowed_hosts_array,
68
- allow_ip_hosts: allow_ip_hosts_value
94
+ referer: StringScrubber.without_query(request.referer),
95
+ x_forwarded_for: request.x_forwarded_for
69
96
  )
70
97
 
71
- # Log the blocked host
72
98
  LogStruct.warn(log_entry)
73
-
74
- # Use pre-defined headers and response if we are only logging or reporting
75
- # Dup the headers so they can be modified by downstream middleware
76
- [FORBIDDEN_STATUS, RESPONSE_HEADERS.dup, [RESPONSE_HTML]]
77
99
  end
78
-
79
- # Merge our response_app into existing host_authorization config to preserve excludes
80
- existing = Rails.application.config.host_authorization
81
- unless existing.is_a?(Hash)
82
- existing = {}
83
- end
84
- existing = existing.dup
85
- existing[:response_app] = response_app
86
- Rails.application.config.host_authorization = existing
87
-
88
- true
89
100
  end
90
101
  end
91
102
  end
@@ -36,6 +36,20 @@ module LogStruct
36
36
  T::Array[Symbol]
37
37
  )
38
38
 
39
+ # With config.exceptions_app = routes, Rails renders an error page by sending the request through the app again
40
+ # as GET /<status>, and keeps the request's own method and path in these headers
41
+ ORIGINAL_METHOD_HEADER = "action_dispatch.original_request_method"
42
+ ORIGINAL_PATH_HEADER = "action_dispatch.original_path"
43
+ # Set when the request's action logged a line with its error, so the error page for it logs none
44
+ ACTION_ERROR_LOGGED_HEADER = "logstruct.action_error_logged"
45
+
46
+ # A Proc, not a Method: sorbet-runtime replaces a method with sig checks on its first call, so a Method taken
47
+ # before then isn't equal to one taken after, and setup couldn't tell it had added the test already
48
+ IGNORE_REPEATED_ERROR_PAGE = T.let(
49
+ ->(event) { Lograge.repeated_error_page?(event) },
50
+ T.proc.params(event: ActiveSupport::Notifications::Event).returns(T::Boolean)
51
+ )
52
+
39
53
  class << self
40
54
  extend T::Sig
41
55
 
@@ -70,7 +84,26 @@ module LogStruct
70
84
  config.lograge.custom_options = lambda do |event|
71
85
  Integrations::Lograge.lograge_default_options(event)
72
86
  end
87
+
88
+ # Lograge detaches Rails' own request lines from ActiveSupport::Notifications. From Rails 8.2 they come
89
+ # from Rails.event, so LogStruct detaches them there.
90
+ config.after_initialize do |app|
91
+ Integrations::Lograge.unsubscribe_rails_request_lines unless app.config.lograge.keep_original_rails_log
92
+ end
73
93
  end
94
+
95
+ # Lograge.setup adds the app's own ignore tests to the same list
96
+ ::Lograge.ignore(IGNORE_REPEATED_ERROR_PAGE) unless ::Lograge.ignore_tests.include?(IGNORE_REPEATED_ERROR_PAGE)
97
+ end
98
+
99
+ # Rails' "Processing by", "Completed 200 OK" and "Rendered" lines, which the request line replaces
100
+ sig { void }
101
+ def unsubscribe_rails_request_lines
102
+ # Earlier Rails versions' subscribers are lograge's to detach
103
+ return unless defined?(::ActiveSupport::EventReporter::LogSubscriber)
104
+
105
+ RailsLogSubscribers.unsubscribe_from_events(::ActionController::LogSubscriber) if defined?(::ActionController::LogSubscriber)
106
+ RailsLogSubscribers.unsubscribe_from_events(::ActionView::LogSubscriber) if defined?(::ActionView::LogSubscriber)
74
107
  end
75
108
 
76
109
  sig { params(event: ActiveSupport::Notifications::Event).returns(T::Hash[Symbol, T.untyped]) }
@@ -81,6 +114,8 @@ module LogStruct
81
114
  :host,
82
115
  :source_ip
83
116
  ).compact
117
+ add_request_address(event.payload[:request], options)
118
+ add_error_page_request(event, options)
84
119
 
85
120
  if event.payload[:params].present?
86
121
  options[:params] = event.payload[:params].except("controller", "action")
@@ -95,6 +130,44 @@ module LogStruct
95
130
  options
96
131
  end
97
132
 
133
+ # Rails' payload has the request but not its address, which Rails' own "Started GET" line logs.
134
+ # A payload's own :source_ip and :host (from append_info_to_payload) take precedence.
135
+ sig { params(request: T.untyped, options: T::Hash[Symbol, T.untyped]).void }
136
+ def add_request_address(request, options)
137
+ return unless request.is_a?(::ActionDispatch::Request)
138
+
139
+ options[:host] ||= request.host
140
+ options[:source_ip] ||= begin
141
+ request.remote_ip
142
+ rescue ::ActionDispatch::RemoteIp::IpSpoofAttackError
143
+ # The forwarding headers disagree, so the address the connection came from
144
+ request.get_header("REMOTE_ADDR")
145
+ end
146
+ end
147
+
148
+ # An error page's line has the method and path of the request it's for, not GET /500
149
+ sig { params(event: ActiveSupport::Notifications::Event, options: T::Hash[Symbol, T.untyped]).void }
150
+ def add_error_page_request(event, options)
151
+ request = event.payload[:request]
152
+ return unless request.is_a?(::ActionDispatch::Request)
153
+
154
+ if request.has_header?(ORIGINAL_PATH_HEADER)
155
+ options[:method] = request.get_header(ORIGINAL_METHOD_HEADER)
156
+ options[:path] = "#{request.script_name}#{request.get_header(ORIGINAL_PATH_HEADER)}"
157
+ elsif event.payload[:exception]
158
+ request.set_header(ACTION_ERROR_LOGGED_HEADER, true)
159
+ end
160
+ end
161
+
162
+ # The error page for an error its action's line logged would log the request a second time. A request whose
163
+ # error came before any action (a routing error) or after it (in a middleware) keeps its error page's line.
164
+ sig { params(event: ActiveSupport::Notifications::Event).returns(T::Boolean) }
165
+ def repeated_error_page?(event)
166
+ request = event.payload[:request]
167
+ request.is_a?(::ActionDispatch::Request) && request.has_header?(ORIGINAL_PATH_HEADER) &&
168
+ request.get_header(ACTION_ERROR_LOGGED_HEADER) == true
169
+ end
170
+
98
171
  # Process headers from the event payload
99
172
  sig { params(event: ActiveSupport::Notifications::Event, options: T::Hash[Symbol, T.untyped]).void }
100
173
  def process_headers(event, options)
@@ -119,9 +192,11 @@ module LogStruct
119
192
  custom_options_proc = LogStruct.config.integrations.lograge_custom_options
120
193
  return unless custom_options_proc&.respond_to?(:call)
121
194
 
122
- # Call the proc with the event and options
123
- # The proc can modify the options hash directly
124
- custom_options_proc.call(event, options)
195
+ # The proc can change options, or return a Hash of fields to add, as Lograge's custom_options do
196
+ result = custom_options_proc.call(event, options)
197
+ return if !result.is_a?(Hash) || result.equal?(options)
198
+
199
+ options.merge!(result.to_h { |key, value| [key.to_s.to_sym, value] })
125
200
  end
126
201
 
127
202
  sig { params(data: T::Hash[T.any(Symbol, String), T.untyped]).returns(Log::Request) }
@@ -152,7 +227,8 @@ module LogStruct
152
227
  params: params,
153
228
  source_ip: normalized_data[:source_ip]&.to_s,
154
229
  user_agent: normalized_data[:user_agent]&.to_s,
155
- referer: normalized_data[:referer]&.to_s,
230
+ # Without its query, which can carry reset tokens and OAuth codes
231
+ referer: StringScrubber.without_query(normalized_data[:referer]&.to_s),
156
232
  host: normalized_data[:host]&.to_s,
157
233
  content_type: normalized_data[:content_type]&.to_s,
158
234
  accept: normalized_data[:accept]&.to_s,
@@ -13,6 +13,7 @@ module LogStruct
13
13
  boot_emitted: false,
14
14
  shutdown_emitted: false,
15
15
  handler_pending_started: false,
16
+ signals_trapped: false,
16
17
  start_info: {
17
18
  mode: nil,
18
19
  puma_version: nil,
@@ -67,8 +68,9 @@ module LogStruct
67
68
  handle_integration_error(e)
68
69
  end
69
70
  begin
70
- %w[TERM INT].each do |sig|
71
- Signal.trap(sig) { emit_shutdown!(sig) }
71
+ unless STATE[:signals_trapped]
72
+ STATE[:signals_trapped] = true
73
+ %w[TERM INT].each { |signal| trap_shutdown_signal(signal) }
72
74
  end
73
75
  rescue => e
74
76
  handle_integration_error(e)
@@ -95,32 +97,30 @@ module LogStruct
95
97
  begin
96
98
  begin
97
99
  require "puma"
98
- rescue => e
99
- handle_integration_error(e)
100
+ rescue LoadError
101
+ # Puma isn't installed (another server, or a command with "server" in its arguments)
102
+ STATE[:installed] = false
103
+ return
100
104
  end
101
- puma_mod = ::Object.const_defined?(:Puma) ? T.unsafe(::Object.const_get(:Puma)) : nil # rubocop:disable Sorbet/ConstantsFromStrings
105
+ puma_mod = ::Object.const_defined?(:Puma) ? ::Object.const_get(:Puma) : nil # rubocop:disable Sorbet/ConstantsFromStrings
102
106
  # rubocop:disable Sorbet/ConstantsFromStrings
103
107
  if puma_mod&.const_defined?(:LogWriter)
104
- T.unsafe(::Object.const_get("Puma::LogWriter")).prepend(LogWriterPatch)
108
+ ::Object.const_get("Puma::LogWriter").prepend(LogWriterPatch)
109
+ end
110
+ if puma_mod&.const_defined?(:ErrorLogger)
111
+ ::Object.const_get("Puma::ErrorLogger").prepend(ErrorLoggerPatch)
105
112
  end
106
113
  if puma_mod&.const_defined?(:Events)
107
- ev = T.unsafe(::Object.const_get("Puma::Events"))
114
+ ev = ::Object.const_get("Puma::Events")
108
115
  ev.prepend(EventsPatch)
109
116
  end
110
- if puma_mod&.const_defined?(:Cluster)
111
- cluster_mod = T.cast(puma_mod.const_get(:Cluster), Module)
112
- if cluster_mod.const_defined?(:Worker)
113
- worker_class = T.cast(cluster_mod.const_get(:Worker), T::Class[T.anything])
114
- worker_class.prepend(ClusterWorkerPatch)
115
- end
116
- end
117
117
  # Patch Rack::Handler::Puma.run to emit lifecycle logs using options
118
118
  if ::Object.const_defined?(:Rack)
119
- rack_mod = T.unsafe(::Object.const_get(:Rack))
119
+ rack_mod = ::Object.const_get(:Rack)
120
120
  if rack_mod.const_defined?(:Handler)
121
- handler_mod = T.unsafe(rack_mod.const_get(:Handler))
121
+ handler_mod = rack_mod.const_get(:Handler)
122
122
  if handler_mod.const_defined?(:Puma)
123
- handler = T.unsafe(handler_mod.const_get(:Puma))
123
+ handler = handler_mod.const_get(:Puma)
124
124
  handler.singleton_class.prepend(RackHandlerPatch)
125
125
  end
126
126
  end
@@ -141,7 +141,8 @@ module LogStruct
141
141
  if defined?(::Rails) && ::Rails.respond_to?(:env) && ::Rails.env.test? && !server_mode
142
142
  raise e
143
143
  else
144
- LogStruct.handle_exception(e, source: Source::Puma)
144
+ # LogStruct's own error: logged in production, so Puma keeps running
145
+ LogStruct.handle_exception(e, source: Source::Internal, context: {integration: :puma})
145
146
  end
146
147
  end
147
148
 
@@ -231,7 +232,7 @@ module LogStruct
231
232
  if (m = l.match(/^\*?\s*Listening on (.+)$/))
232
233
  si = T.cast(STATE[:start_info], T::Hash[Symbol, T.untyped])
233
234
  list = T.cast(si[:listening], T::Array[T.untyped])
234
- address = T.must(m[1])
235
+ address = m[1].to_s
235
236
  list << address unless list.include?(address)
236
237
  # Emit started when we see the first listening address
237
238
  if !STATE[:started_emitted]
@@ -313,6 +314,31 @@ module LogStruct
313
314
  # Only use LogStruct; SemanticLogger routes to STDOUT in test
314
315
  end
315
316
 
317
+ # Logs the shutdown when the signal arrives, then runs the handler that was there before
318
+ # (Puma's, which stops the server gracefully), or does what the signal does by default
319
+ sig { params(signal: String).void }
320
+ def trap_shutdown_signal(signal)
321
+ previous = T.let(nil, T.untyped)
322
+ previous = Signal.trap(signal) do |signal_number|
323
+ emit_shutdown!(signal)
324
+ run_previous_signal_handler(previous, signal, signal_number)
325
+ end
326
+ end
327
+
328
+ sig { params(previous: T.untyped, signal: String, signal_number: T.untyped).void }
329
+ def run_previous_signal_handler(previous, signal, signal_number)
330
+ if previous.respond_to?(:call)
331
+ (previous.respond_to?(:arity) && previous.arity == 0) ? previous.call : previous.call(signal_number)
332
+ elsif previous == "EXIT"
333
+ Kernel.exit
334
+ elsif previous.nil? || previous == "DEFAULT" || previous == "SYSTEM_DEFAULT"
335
+ # Put back the default and send the signal again, so the process ends as it would have
336
+ Signal.trap(signal, previous || "DEFAULT")
337
+ Process.kill(signal, Process.pid)
338
+ end
339
+ # "IGNORE": nothing more to do
340
+ end
341
+
316
342
  sig { params(_message: String).void }
317
343
  def emit_shutdown!(_message)
318
344
  return if STATE[:shutdown_emitted]
@@ -366,7 +392,7 @@ module LogStruct
366
392
  return nil
367
393
  end
368
394
  if ::Kernel.instance_variables.include?(:@stdout)
369
- io = T.unsafe(::Kernel.instance_variable_get(:@stdout))
395
+ io = ::Kernel.instance_variable_get(:@stdout)
370
396
  return io.puts(msg)
371
397
  end
372
398
  super
@@ -379,6 +405,34 @@ module LogStruct
379
405
  end
380
406
  end
381
407
 
408
+ # Puma::ErrorLogger writes request errors (an exception the app didn't handle, a malformed
409
+ # request, an SSL error) as plain text to STDERR, with the request's query string. Log them
410
+ # as structured lines, with the method and path but not the query string, which LogStruct
411
+ # can't filter there.
412
+ module ErrorLoggerPatch
413
+ extend T::Sig
414
+
415
+ sig { params(options: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
416
+ def info(options = {})
417
+ error = options[:error]
418
+ return super unless error.is_a?(Exception)
419
+
420
+ env = options[:req]&.env
421
+ ::LogStruct.error(::LogStruct::Log::Puma::Error.new(
422
+ error_class: error.class,
423
+ message: error.message,
424
+ backtrace: error.backtrace,
425
+ error_context: options[:text]&.to_s,
426
+ http_method: env&.dig("REQUEST_METHOD"),
427
+ path: env && (env["REQUEST_PATH"] || env["PATH_INFO"])
428
+ ))
429
+ nil
430
+ rescue => e
431
+ ::LogStruct::Integrations::Puma.handle_integration_error(e)
432
+ super
433
+ end
434
+ end
435
+
382
436
  # Patch Puma::Events as a fallback for some versions where Events handles output
383
437
  module EventsPatch
384
438
  extend T::Sig
@@ -390,20 +444,6 @@ module LogStruct
390
444
  end
391
445
  end
392
446
 
393
- module ClusterWorkerPatch
394
- extend T::Sig
395
-
396
- sig { returns(T.untyped) }
397
- def run
398
- begin
399
- ::SemanticLogger.reopen
400
- rescue => e
401
- ::LogStruct::Integrations::Puma.handle_integration_error(e)
402
- end
403
- super
404
- end
405
- end
406
-
407
447
  # Hook Rack::Handler::Puma.run to emit structured started/shutdown
408
448
  module RackHandlerPatch
409
449
  extend T::Sig
@@ -0,0 +1,35 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "error_lines"
5
+
6
+ module LogStruct
7
+ module Integrations
8
+ module RackErrorHandler
9
+ # Prepended to ActionDispatch::DebugExceptions. For an error the app didn't handle, Rails logs a
10
+ # block of text (the error, its source code and its trace) before ShowExceptions renders the
11
+ # error page. LogStruct writes an error line instead.
12
+ module DebugExceptionsLogging
13
+ extend T::Sig
14
+ extend T::Helpers
15
+
16
+ requires_ancestor { ::ActionDispatch::DebugExceptions }
17
+
18
+ private
19
+
20
+ sig { params(request: ::ActionDispatch::Request, wrapper: T.untyped).returns(T.untyped) }
21
+ def log_error(request, wrapper)
22
+ return super unless ErrorLines.enabled?
23
+ # As Rails does: config.action_dispatch.log_rescued_responses = false leaves out the errors
24
+ # Rails turns into a response of their own (404 for a routing error, 422 for CSRF)
25
+ return if !request.get_header("action_dispatch.log_rescued_responses") && wrapper.rescue_response?
26
+
27
+ ErrorLines.log_unhandled(request, wrapper)
28
+ rescue => e
29
+ LogStruct.handle_exception(e, source: Source::Internal, context: {integration: :rack_error_handler})
30
+ super
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end