logtail-rails 0.2.14 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbc8fb02a00988c89f406be59ac925eba90048afc392ac52c5efca571f835271
-  data.tar.gz: 8bb940e1915fd178e7cc6854415f345101ec74b9e441173464624ba94ac7ea5a
+  metadata.gz: 2c5828f2141c9b70b0f47591b7dcf8855a648c0297477637fd7029cfd5e39a8d
+  data.tar.gz: 0a88a29844cdbbc76a20a827f0575fc3f3aa4befe32dde404315bb55a733cfa2
 SHA512:
-  metadata.gz: e75cc0a7bd43ea6b8cccac45f8f4b3c74e0acea9b833ac14c6b77c1e5ad63fcbc35dc22d9a32b58a6ba41e7ce3b5f964a20b7a0bdc6df2ebc1a6bdfdb84194d0
-  data.tar.gz: 7b675f409e820c32dd256371157b155f5b7a902e91e188fe6be389bb139f9ba33797898c4ed43e407928fab5ac4237fd839c123edd3e45c7378467a787e8f9de
+  metadata.gz: 9238c31e5e7945f99fdc3acae8a7006f6f0568e3e2269469a0498da0d5010718d2a3dc884bfd64cc471be0c7a54283e32ff327f8fa1dd11d70b76d1baf84dc2c
+  data.tar.gz: 76f5185abcdc9efafbe564d65a543cb924dcfb75326686645f346a7e30201b053cad5380701f8abe3d0ed01db85f0a54b631522dca449bf4aba2164967f6d840

data/README.md

@@ -14,6 +14,8 @@ Experience SQL-compatible structured log management based on ClickHouse. [Learn
 
 [Getting started ⇗](https://betterstack.com/docs/logs/ruby-and-rails/#logging-from-rails)
 
+[Example project](example-project/README.md): logging, structured data and [adding context to every log line of a request](example-project/README.md#context).
+
 ## Need help?
 Please let us know at [hello@betterstack.com](mailto:hello@betterstack.com). We're happy to help!
 

data/example-project/README.md

@@ -185,12 +185,53 @@ This will generate the following JSON output:
 }
 ```
 
-We will automatically add the information about the current user to each log if you're using Ruby on Rails and the Devise gem.
+We will automatically add the information about the current user to each log if you're using Ruby on Rails with Devise (or any other Warden-based authentication) or with Clearance.
 
-If you're not using Devise or you want to log some additional information for every request your Rails app handles, you can easily implement this using Rails' `around_action` in your application controller. A simple implementation could look like this:
+### Context for every log line of a request
+
+The request and response lines (`Started GET "/" for 127.0.0.1` and `Completed 200 OK in 12.3ms`) are logged by a Rack middleware, before the request reaches your controller. Context set from inside a controller is therefore attached to the logs from the controller action on, but not to these two lines. To attach context to every log line of a request, set it in a Rack middleware instead.
+
+The user context from Devise or Clearance is set this way, by the `Logtail::Integrations::Rack::UserContext` middleware. If you authenticate differently, give it a lambda that finds the user in the Rack environment and returns a hash, or `nil` when nobody is signed in:
 
 ```ruby
-class ApplicationController < ActionController::Base
+# config/initializers/logtail.rb
+Logtail::Integrations::Rack::UserContext.custom_user_hash = lambda do |rack_env|
+  user_id = rack_env["rack.session"]["user_id"]
+  user = User.find_by(id: user_id) if user_id
+  user && { id: user.id, email: user.email }
+end
+```
+
+The hash is logged as `context.user` on every log line of the request, including the request and response lines. The lambda runs on every request, so keep it cheap. It can read anything from the Rack environment, for example an API token from `rack_env["HTTP_AUTHORIZATION"]`.
+
+For any other per-request context (a tenant, an API client, a GraphQL operation name), write your own middleware that wraps the request in `Logtail.with_context` and insert it before the middleware that logs the request and response lines:
+
+```ruby
+# config/initializers/logtail.rb
+class LogtailTenantContext
+  def initialize(app)
+    @app = app
+  end
+
+  def call(env)
+    tenant = env["HTTP_X_TENANT"]
+    return @app.call(env) unless tenant
+
+    Logtail.with_context(tenant: { name: tenant }) { @app.call(env) }
+  end
+end
+
+Rails.application.config.middleware.insert_before Logtail::Integrations::Rack::HTTPEvents, LogtailTenantContext
+```
+
+Inserting it before `Logtail::Integrations::Rack::HTTPEvents` also places it after your authentication middleware (such as Warden), so the session and the signed-in user are already available in `env`.
+
+### Context from a controller
+
+If the user is only known once your controller runs (for example, you authenticate from a token in a `before_action`), you can still set the context with Rails' `around_action`. It covers the logs from the controller action and everything it calls, but not the request and response lines, which are logged before the action runs. A simple implementation could look like this:
+
+```ruby
+class ApplicationController < ActionController::Base # or ActionController::API
   around_action :with_logtail_context
 
   private
@@ -202,7 +243,7 @@ class ApplicationController < ActionController::Base
         yield
       end
     end
-    
+
     def user_context
       Logtail::Contexts::User.new(
         id: current_user.id,

data/lib/logtail-rails/action_view/log_subscriber/logtail_log_subscriber.rb

@@ -76,9 +76,12 @@ module Logtail
               super
 
               if ::Rails::VERSION::MAJOR > 7 || ::Rails::VERSION::MAJOR == 7 && ::Rails::VERSION::MINOR >= 1
-                # Clean extra listeners subscribed in parent's attach_to method
-                ::ActiveSupport::Notifications.notifier.listeners_for("render_template.action_view")
-                  .concat(::ActiveSupport::Notifications.notifier.listeners_for("render_layout.action_view")).flatten
+                # Clean extra listeners subscribed in parent's attach_to method. Since Rails 7.2 they count as
+                # silenced while ActionView::Base.logger is nil, as it is at boot, and listeners_for skips those.
+                # all_listeners_for returns the notifier's cached array, so don't concat onto it.
+                notifier = ::ActiveSupport::Notifications.notifier
+                %w(render_template.action_view render_layout.action_view)
+                  .flat_map { |name| notifier.respond_to?(:all_listeners_for) ? notifier.all_listeners_for(name) : notifier.listeners_for(name) }
                   .filter { |listener| listener.delegate.class == ::ActionView::LogSubscriber::Start }
                   .each { |listener| ActiveSupport::Notifications.unsubscribe(listener) }
               end

data/lib/logtail-rails/config.rb

@@ -10,3 +10,22 @@ Logtail::Config.instance.define_singleton_method(:logrageify!) do
   integrations.active_record.silence = true
   integrations.rack.http_events.collapse_into_single_event = true
 end
+
+module Logtail
+  class Config
+    # Turns the Rails integration on or off, it's on by default. Set it in `config/application.rb`
+    # or `config/environments/*.rb`, see {Logtail::Frameworks::Rails::Railtie} for what it turns off.
+    #
+    # @example Keep the standard Rails logging in development
+    #   # config/environments/development.rb
+    #   config.logtail.enabled = false
+    def enabled=(value)
+      @enabled = value
+    end
+
+    # Accessor method for {#enabled=}
+    def enabled?
+      @enabled != false
+    end
+  end
+end

data/lib/logtail-rails/error_event.rb

@@ -18,19 +18,17 @@ module Logtail
         # We determine this when the app loads to avoid the overhead on a per request basis.
         EXCEPTION_WRAPPER_TAKES_CLEANER = defined?(::ActionDispatch::ExceptionWrapper) &&
           !::ActionDispatch::ExceptionWrapper.instance_methods.include?(:env)
+        # config.action_dispatch.log_rescued_responses (Rails 7.0+) and
+        # config.action_dispatch.debug_exception_log_level (Rails 7.1+), as ActionDispatch::DebugExceptions
+        # reads them. This gem silences its logging and logs the exception here instead.
+        LOG_RESCUED_RESPONSES_KEY = "action_dispatch.log_rescued_responses".freeze
+        DEBUG_EXCEPTION_LOG_LEVEL_KEY = "action_dispatch.debug_exception_log_level".freeze
 
         def call(env)
           begin
             status, headers, body = @app.call(env)
           rescue Exception => exception
-            Config.instance.logger.fatal do
-              backtrace = extract_backtrace(env, exception)
-              Events::Error.new(
-                name: exception.class.name,
-                error_message: exception.message,
-                backtrace: backtrace
-              )
-            end
+            log_exception(env, exception)
 
             raise exception
           end
@@ -38,6 +36,30 @@ module Logtail
 
         private
 
+        # Never raises, so that the exception of the app is the one that propagates.
+        def log_exception(env, exception)
+          return if !log_exception?(env, exception)
+
+          Config.instance.logger.add(env[DEBUG_EXCEPTION_LOG_LEVEL_KEY] || ::Logger::FATAL) do
+            backtrace = extract_backtrace(env, exception)
+            Events::Error.new(
+              name: exception.class.name,
+              error_message: exception.message,
+              backtrace: backtrace
+            )
+          end
+        rescue StandardError => e
+          Config.instance.debug { "#{self.class.name} could not log #{exception.class}: #{e.inspect}" }
+        end
+
+        # Like DebugExceptions, skip rescued responses such as ActiveRecord::RecordNotFound when
+        # log_rescued_responses is false. Older Rails versions don't set it and log every exception.
+        def log_exception?(env, exception)
+          return true if !env.key?(LOG_RESCUED_RESPONSES_KEY) || env[LOG_RESCUED_RESPONSES_KEY]
+
+          !::ActionDispatch::ExceptionWrapper.rescue_responses.key?(exception.class.name)
+        end
+
         # Rails provides a backtrace cleaner, so we use it here.
         def extract_backtrace(env, exception)
           if defined?(::ActionDispatch::ExceptionWrapper)

data/lib/logtail-rails/event_log_subscriber.rb

@@ -6,7 +6,7 @@ module Logtail
       # them with all their data to Rails.logger, which sends them to Better Stack.
       class EventLogSubscriber
         # Rails logger instance
-        attr_reader :logger
+        attr_accessor :logger
 
         # Log level to use for logging events
         mattr_accessor :log_level, default: :info
@@ -14,6 +14,17 @@ module Logtail
         # Allows to disable the subscriber
         mattr_accessor :enabled, default: true
 
+        # Subscribes to Rails.event once. Later calls only hand the events to the new logger, so that
+        # calling Logtail::Logger.create_default_logger again doesn't log every event twice.
+        def self.subscribe(logger)
+          if @subscriber
+            @subscriber.logger = logger
+          else
+            @subscriber = new(logger)
+            ::Rails.event.subscribe(@subscriber)
+          end
+        end
+
         # Initialize the subscriber with a logger instance
         def initialize(logger)
           @logger = logger

data/lib/logtail-rails/logger.rb

@@ -49,6 +49,10 @@ module Logtail
     def self.create_logger(*io_devices_and_loggers)
       logger = Logtail::Logger.new(*io_devices_and_loggers)
 
+      # Rails applies config.log_level to Rails.logger while booting, but not to a logger added with broadcast_to later
+      log_level = Rails.application.config.log_level if ENV['LOG_LEVEL'].blank? && Rails.application
+      logger.level = ::ActiveSupport::Logger.const_get(log_level.to_s.upcase) if log_level
+
       tagged_logging_supported = Rails::VERSION::MAJOR >= 7 || Rails::VERSION::MAJOR == 6 && Rails::VERSION::MINOR >= 1
       logger = ::ActiveSupport::TaggedLogging.new(logger) if tagged_logging_supported
 
@@ -61,10 +65,15 @@ module Logtail
       options[:ingesting_port] ||= options[:telemetry_port] if options[:telemetry_port].present?
       options[:ingesting_scheme] ||= options[:telemetry_scheme] if options[:telemetry_scheme].present?
 
-      if ENV['LOGTAIL_SKIP_LOGS'].blank? && !Rails.env.test?
-        io_device = Logtail::LogDevices::HTTP.new(source_token, options)
-      else
+      if ENV['LOGTAIL_SKIP_LOGS'].present? || Rails.env.test?
+        io_device = STDOUT
+      elsif source_token.blank?
+        # Don't break booting the app without the token, e.g. for `assets:precompile` in a Docker build
+        @blank_source_token = true
         io_device = STDOUT
+      else
+        io_device = Logtail::LogDevices::HTTP.new(source_token, options)
+        @better_stack_logger_created = true
       end
 
       logger = self.create_logger(io_device)
@@ -79,9 +88,16 @@ module Logtail
       end
 
       # For Rails 8.1 and above, subscribe to the event system
-      Rails.event.subscribe(Logtail::Integrations::Rails::EventLogSubscriber.new(logger)) if Rails.respond_to?(:event)
+      Logtail::Integrations::Rails::EventLogSubscriber.subscribe(logger) if Rails.respond_to?(:event)
 
       logger
     end
+
+    class << self
+      # What {create_default_logger} did, Logtail::Frameworks::Rails::Railtie warns about it once the app has booted
+      #
+      # @private
+      attr_reader :blank_source_token, :better_stack_logger_created
+    end
   end
 end

data/lib/logtail-rails/railtie.rb

@@ -13,6 +13,26 @@ module Logtail
           Logtail::Config.instance.logger = Proc.new { ::Rails.logger }
         end
 
+        # `config.logtail.enabled = false` turns the integration off. The environment files are loaded
+        # by now, and Rails hasn't set up Rails.logger from config.logger yet.
+        initializer(:logtail_enabled, after: :load_environment_config, before: :initialize_logger) do |app|
+          next if app.config.logtail.enabled?
+
+          # Before the :logtail initializer below picks the middlewares
+          Integrations::Rails.enabled = false
+
+          # The setup docs create the logger in config/application.rb, i.e. in every environment. Log to the
+          # default log file instead, with a Logtail::Logger so that `logger.info("message", key: value)` works.
+          if app.config.logger.is_a?(Logtail::Logger)
+            log_file = app.paths["log"].first
+            FileUtils.mkdir_p(File.dirname(log_file))
+            # Opened like Rails opens it: Logger only writes its "# Logfile created" header into files it creates
+            file = File.open(log_file, "a")
+            file.binmode
+            app.config.logger = Logtail::Logger.create_logger(file)
+          end
+        end
+
         # Must be loaded after initializers so that we respect any Logtail configuration set
         initializer(:logtail, before: :build_middleware_stack, after: :load_config_initializers) do
           Integrations::Rails.integrate!
@@ -23,6 +43,31 @@ module Logtail
             config.app_middleware.use middleware_class
           end
         end
+
+        # Warns about problems with the logger create_default_logger created, unless the integration is turned off.
+        # Registered here, so that it runs after the app's own after_initialize blocks, which may still broadcast to it.
+        initializer(:logtail_logger_check, after: :load_config_initializers) do |app|
+          app.config.after_initialize do
+            next unless Integrations::Rails.enabled?
+
+            if Logtail::Logger.blank_source_token
+              Kernel.warn("Logtail: the source token passed to Logtail::Logger.create_default_logger is blank, logging to STDOUT instead of sending logs to Better Stack.")
+              next
+            end
+            next unless Logtail::Logger.better_stack_logger_created
+
+            # A `config.logger = ...` line after config/application.rb, like the one generated in
+            # config/environments/production.rb, replaces the Better Stack logger without any error
+            loggers = ::Rails.logger.respond_to?(:broadcasts) ? ::Rails.logger.broadcasts : [::Rails.logger]
+            next if loggers.any? { |logger| logger.is_a?(Logtail::Logger) }
+
+            # Before Rails 7.1, broadcasting extends the logger with an anonymous module, which hides the target
+            extended_modules = ::Rails.logger.singleton_class.included_modules - ::Rails.logger.class.included_modules
+            next if extended_modules.any? { |mod| mod.name.nil? }
+
+            Kernel.warn("Logtail: Rails.logger isn't the Better Stack logger created by Logtail::Logger.create_default_logger, and doesn't broadcast to it, so your logs don't reach Better Stack. Most likely config.logger is set again later, e.g. in config/environments/production.rb.")
+          end
+        end
       end
     end
   end

data/lib/logtail-rails/version.rb

@@ -1,7 +1,7 @@
 module Logtail
   module Integrations
     module Rails
-      VERSION = "0.2.14"
+      VERSION = "0.3.0"
     end
   end
 end

data/lib/logtail-rails.rb

@@ -39,11 +39,18 @@ module Logtail
       def self.integrate!
         return false if !enabled?
 
+        # Log the status Rails responds with when the app raises, the way ShowExceptions
+        # determines it: from config.action_dispatch.rescue_responses, 500 by default.
+        Logtail::Integrations::Rack::HTTPEvents.status_for_exception = lambda do |exception|
+          ::ActionDispatch::ExceptionWrapper.new(nil, exception).status_code
+        end
+
         ActionController.integrate!
         ActionDispatch.integrate!
         ActionView.integrate!
         ActiveRecord.integrate!
         RackLogger.integrate!
+        filter_parameters_in_urls(::Rails.application.config.filter_parameters)
       end
 
       def self.enabled=(value)
@@ -65,6 +72,16 @@ module Logtail
         @middlewares ||= [Logtail::Integrations::Rack::HTTPContext, SessionContext, Logtail::Integrations::Rack::UserContext,
           Logtail::Integrations::Rack::HTTPEvents, Logtail::Integrations::Rails::ErrorEvent].select(&:enabled?)
       end
+
+      # Filters the app's filter_parameters from the query strings and the Referer and Location
+      # URLs that Rack::HTTPEvents logs, unless the app customised its query_string_filters.
+      # Procs are left out: they rewrite values, while query string filters match names.
+      def self.filter_parameters_in_urls(filter_parameters)
+        http_events = Logtail::Integrations::Rack::HTTPEvents
+        return if http_events.query_string_filters != http_events::DEFAULT_QUERY_STRING_FILTERS
+
+        http_events.query_string_filters = filter_parameters.reject { |filter| filter.is_a?(Proc) }
+      end
     end
   end
 end

data/logtail-rails.gemspec

@@ -27,8 +27,8 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "logtail", "~> 0.1", ">= 0.1.14"
-  spec.add_runtime_dependency "logtail-rack", "~> 0.1"
+  spec.add_runtime_dependency "logtail", "~> 0.1", ">= 0.1.21"
+  spec.add_runtime_dependency "logtail-rack", "~> 0.2", ">= 0.2.9"
 
   spec.add_runtime_dependency 'activerecord', '>= 5.0.0'
   spec.add_runtime_dependency 'railties', '>= 5.0.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logtail-rails
 version: !ruby/object:Gem::Version
-  version: 0.2.14
+  version: 0.3.0
 platform: ruby
 authors:
 - Better Stack
@@ -18,7 +18,7 @@ dependencies:
         version: '0.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.14
+        version: 0.1.21
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,21 +28,27 @@ dependencies:
         version: '0.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.14
+        version: 0.1.21
 - !ruby/object:Gem::Dependency
   name: logtail-rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.9
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement