scout_apm_logging 0.0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 198bf309d17f0abfcf2c33f820a2d44e1e92ef5af9e321ee9fbaacfec256f032
-  data.tar.gz: 2a09b14a45e702fe1e4f4caee6ff89475e14971ba9edad41ae5cf3c72ab97850
+  metadata.gz: bb4dc1a11fe63b4c983636958676d8ca7e76bb04c5281bdabd272373deaf83e5
+  data.tar.gz: 70809502cbbfb93a9cc73e26329f4e52b57008f51f9e34b3769fdfb17b2d46f4
 SHA512:
-  metadata.gz: ebd30ceac50e2f14f3693946e8dd883112cc502513678b3a9963e253f2622d1ea488c2a66bce01f3e20726d7c6f08f61392cde6f22f1d9d0412987780f6399fa
-  data.tar.gz: c0cfeb3cfbed2628772550b18b1a2f7965a144fafea14846ed65936043ae283b48cef6ba0109a05dfe9df3b826080d0f24676c0957b3909311cae12098b1e2de
+  metadata.gz: b3a17d497a14d952294dd23ce5eac8605f7cee5da807ba53cc312acafbe314c33a303d75d97ac828f3ea1aba40e733da1104aa655866121e7f3c149afc9935cb
+  data.tar.gz: 3515ffc8593bc564318859e0fb7340f82c2c066509a82411091b8e5e21bb4abcb223e1318d83b76a12ac11986db22bf4abaec5f511d6c07bdcaa5cc32742b7fd

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## 0.0.3
+* **Feature**: Add support for TaggedLogging.
+* Add ability to customize file logger size. Increase default size to 10MiB.
+* Fix an issue with removing of the monitor process when Rails workers exited, which was only intended for when the main process exits.
+* Fix an issue with the known monitored logs state being removed on port flushing.

data/Rakefile

@@ -1,3 +1,5 @@
+require "bundler/gem_tasks"
+
 task :default => :test
 
 task :test do

data/lib/scout_apm/logging/config.rb

@@ -54,6 +54,7 @@ module ScoutApm
         logs_config
         logs_reporting_endpoint
         logs_proxy_log_dir
+        logs_log_file_size
         manager_lock_file
         monitor_pid_file
         monitor_state_file
@@ -72,6 +73,7 @@ module ScoutApm
       SETTING_COERCIONS = {
         'logs_monitor' => BooleanCoercion.new,
         'logs_monitored' => JsonCoercion.new,
+        'logs_log_file_size' => IntegerCoercion.new,
         'monitor_interval' => IntegerCoercion.new,
         'monitor_interval_delay' => IntegerCoercion.new,
         'health_check_port' => IntegerCoercion.new
@@ -227,8 +229,9 @@ module ScoutApm
           'log_level' => 'info',
           'logs_monitored' => [],
           'logs_capture_level' => 'debug',
-          'logs_reporting_endpoint' => 'https://otlp.telemetryhub.com:4317',
+          'logs_reporting_endpoint' => 'https://otlp.scoutotel.com:4317',
           'logs_proxy_log_dir' => '/tmp/scout_apm/logs/',
+          'logs_log_file_size' => 1024 * 1024 * 10,
           'manager_lock_file' => '/tmp/scout_apm/monitor_lock_file.lock',
           'monitor_pid_file' => '/tmp/scout_apm/scout_apm_log_monitor.pid',
           'monitor_state_file' => '/tmp/scout_apm/scout_apm_log_monitor_state.json',

data/lib/scout_apm/logging/loggers/capture.rb

@@ -2,8 +2,13 @@
 
 require 'logger'
 
-require_relative './swap'
+require_relative './formatter'
+require_relative './logger'
 require_relative './proxy'
+require_relative './swaps/rails'
+require_relative './swaps/sidekiq'
+require_relative './swaps/scout'
+require_relative './patches/tagged_logging'
 
 module ScoutApm
   module Logging
@@ -12,33 +17,57 @@ module ScoutApm
       class Capture
         attr_reader :context
 
+        KNOWN_LOGGERS = [
+          Swaps::Rails,
+          Swaps::Sidekiq,
+          Swaps::Scout
+        ].freeze
+
         def initialize(context)
           @context = context
         end
 
-        def capture_log_locations! # rubocop:disable Metrics/AbcSize
-          logger_instances << Rails.logger if defined?(Rails)
-          logger_instances << Sidekiq.logger if defined?(Sidekiq)
-          logger_instances << ObjectSpace.each_object(::ScoutTestLogger).to_a if defined?(::ScoutTestLogger)
+        def setup!
+          return unless context.config.value('logs_monitor')
 
-          # Swap in our logger for each logger instance, in conjunction with the original class.
-          updated_log_locations = logger_instances.compact.flatten.map do |logger|
-            swapped_in_location(logger)
-          end
+          create_proxy_log_dir!
 
-          context.config.state.add_log_locations!(updated_log_locations)
+          add_logging_patches!
+          capture_and_swap_log_locations!
         end
 
         private
 
-        def logger_instances
-          @logger_instances ||= []
+        def create_proxy_log_dir!
+          Utils.ensure_directory_exists(context.config.value('logs_proxy_log_dir'))
+        end
+
+        def add_logging_patches!
+          # We can't swap out the logger similar to that of Rails and Sidekiq, as
+          # the TaggedLogging logger is dynamically generated.
+          return unless Rails.logger.respond_to?(:tagged)
+
+          ::ActiveSupport::TaggedLogging.prepend(Patches::TaggedLogging)
+
+          # Re-extend TaggedLogging to verify the patch is be applied.
+          # This appears to be an issue in Ruby 2.7 with the broadcast logger.
+          ruby_version = Gem::Version.new(RUBY_VERSION)
+          isruby27 = (ruby_version >= Gem::Version.new('2.7') && ruby_version < Gem::Version.new('3.0'))
+          return unless isruby27 && Rails.logger.respond_to?(:broadcasts)
+
+          Rails.logger.broadcasts.each do |logger|
+            logger.extend ::ActiveSupport::TaggedLogging
+          end
         end
 
-        def swapped_in_location(log_instance)
-          swap = Swap.new(context, log_instance)
-          swap.update_logger!
-          swap.log_location
+        def capture_and_swap_log_locations!
+          # We can move this to filter_map when our lagging version is Ruby 2.7
+          updated_log_locations = KNOWN_LOGGERS.map do |logger|
+            logger.new(context).update_logger! if logger.present?
+          end
+          updated_log_locations.compact!
+
+          context.config.state.add_log_locations!(updated_log_locations)
         end
       end
     end

data/lib/scout_apm/logging/loggers/logger.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'logger'
-
 module ScoutApm
   module Logging
     module Loggers
@@ -12,8 +10,6 @@ module ScoutApm
       class Logger
         attr_reader :context, :log_instance
 
-        # 1 MiB
-        LOG_SIZE = 1024 * 1024
         # 1 log file
         LOG_AGE = 1
 
@@ -23,13 +19,16 @@ module ScoutApm
         end
 
         def create_logger!
-          # Defaults are 7 files with 10 MiB.
           # We create the file in order to prevent a creation header log.
           File.new(determine_file_path, 'w+') unless File.exist?(determine_file_path)
-          new_logger = FileLogger.new(determine_file_path, LOG_AGE, LOG_SIZE)
-          # Ruby's Logger handles a lot of the coercion itself.
-          new_logger.level = context.config.value('logs_capture_level')
-          new_logger
+          log_size = context.config.value('logs_log_file_size')
+
+          FileLogger.new(determine_file_path, LOG_AGE, log_size).tap do |logger|
+            # Ruby's Logger handles a lot of the coercion itself.
+            logger.level = context.config.value('logs_capture_level')
+            # Add our custom formatter to the logger.
+            logger.formatter = Formatter.new
+          end
         end
 
         def determine_file_path # rubocop:disable Metrics/AbcSize

data/lib/scout_apm/logging/loggers/patches/tagged_logging.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative '../logger'
+require_relative '../formatter'
+require_relative '../proxy'
+
+module ScoutApm
+  module Logging
+    module Loggers
+      module Patches
+        # Patches TaggedLogging to work with our loggers.
+        module TaggedLogging
+          def tagged(*tags) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+            super(*tags) unless (self == ::Rails.logger && is_a?(ScoutApm::Logging::Loggers::Proxy)) ||
+                                (::Rails.logger.respond_to?(:broadcasts) && ::Rails.logger.broadcasts.include?(self))
+
+            if is_a?(ScoutApm::Logging::Loggers::Proxy)
+              if block_given?
+                # We skip the first logger to prevent double tagging when calling formatter.tagged
+                loggers = @loggers[1..]
+                pushed_counts = extend_and_push_tags(loggers, *tags)
+
+                formatter.tagged(*tags) { yield self }.tap do
+                  logger_pop_tags(loggers, pushed_counts)
+                end
+              else
+                loggers = instance_variable_get(:@loggers)
+
+                new_loggers = create_cloned_extended_loggers(loggers, nil, *tags)
+
+                self.clone.tap do |cp| # rubocop:disable Style/RedundantSelf
+                  cp.instance_variable_set(:@loggers, new_loggers)
+                end
+              end
+            elsif block_given?
+              # We skip the first logger to prevent double tagging when calling formatter.tagged
+              loggers = ::Rails.logger.broadcasts[1..]
+              pushed_counts = extend_and_push_tags(loggers, *tags)
+
+              formatter.tagged(*tags) { yield self }.tap do
+                logger_pop_tags(loggers, pushed_counts)
+              end
+            else
+              broadcasts = ::Rails.logger.broadcasts
+
+              tagged_loggers = broadcasts.select { |logger| logger.respond_to?(:tagged) }
+              file_logger = broadcasts.find { |logger| logger.is_a?(Loggers::FileLogger) }
+              loggers = tagged_loggers << file_logger
+
+              current_tags = tagged_loggers.first.formatter.current_tags
+
+              new_loggers = create_cloned_extended_loggers(loggers, current_tags, *tags)
+              Proxy.create_with_loggers(*new_loggers)
+            end
+          end
+
+          def create_cloned_extended_loggers(loggers, current_tags = nil, *tags)
+            loggers.map do |logger|
+              logger_current_tags = if current_tags
+                                      current_tags
+                                    elsif logger.formatter.respond_to?(:current_tags)
+                                      logger.formatter.current_tags
+                                    else
+                                      []
+                                    end
+
+              ::ActiveSupport::TaggedLogging.new(logger).tap do |new_logger|
+                if defined?(::ActiveSupport::TaggedLogging::LocalTagStorage)
+                  new_logger.formatter.extend ::ActiveSupport::TaggedLogging::LocalTagStorage
+                end
+                new_logger.push_tags(*logger_current_tags, *tags)
+              end
+            end
+          end
+
+          def extend_and_push_tags(loggers, *tags)
+            loggers.map do |logger|
+              logger.formatter.extend ::ActiveSupport::TaggedLogging::Formatter unless logger.formatter.respond_to?(:tagged)
+
+              logger.formatter.push_tags(tags).size
+            end
+          end
+
+          def logger_pop_tags(loggers, pushed_counts)
+            loggers.map.with_index do |logger, index|
+              logger.formatter.pop_tags(pushed_counts[index])
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/loggers/proxy.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
-require 'logger'
-
 module ScoutApm
   module Logging
     module Loggers
       # Holds both the original application logger and the new one. Relays commands to both.
       class Proxy
+        def self.create_with_loggers(*loggers)
+          new.tap do |proxy_logger|
+            loggers.each { |logger| proxy_logger.add(logger) }
+          end
+        end
+
         def initialize
           @loggers = []
         end

data/lib/scout_apm/logging/loggers/swaps/rails.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    module Loggers
+      module Swaps
+        # Swaps in our logger for the Rails logger.
+        class Rails
+          attr_reader :context
+
+          def self.present?
+            defined?(::Rails) && ::Rails.logger
+          end
+
+          def initialize(context)
+            @context = context
+          end
+
+          def update_logger!
+            # In Rails 7.1, broadcast logger was added which allows sinking to multiple IO devices.
+            if defined?(::ActiveSupport::BroadcastLogger) && log_instance.is_a?(::ActiveSupport::BroadcastLogger)
+              add_logger_to_broadcast!
+            else
+              swap_in_proxy_logger!
+            end
+
+            new_log_location
+          end
+
+          private
+
+          def log_instance
+            ::Rails.logger
+          end
+
+          def new_file_logger
+            @new_file_logger ||= Loggers::Logger.new(context, log_instance).create_logger!
+          end
+
+          # Eseentially creates the original logger.
+          def original_logger
+            # We can use the previous logdev. log_device will continuously call write
+            # through the devices until the logdev (@dev) is an IO device other than logdev:
+            # https://github.com/ruby/ruby/blob/master/lib/logger/log_device.rb#L42
+            # Log device holds the configurations around shifting too.
+            original_logdevice = log_instance.instance_variable_get(:@logdev)
+
+            ::Logger.new(original_logdevice).tap do |logger|
+              logger.formatter = log_instance.formatter
+            end
+          end
+
+          def new_log_location
+            new_file_logger.instance_variable_get(:@logdev).filename
+          end
+
+          def add_logger_to_broadcast!
+            log_instance.broadcast_to(new_file_logger)
+          end
+
+          def swap_in_proxy_logger!
+            # First logger needs to be the original logger for the return value of relayed calls.
+            proxy_logger = Proxy.create_with_loggers(original_logger, new_file_logger)
+            proxy_logger.extend ::ActiveSupport::TaggedLogging if log_instance.respond_to?(:tagged)
+
+            ::Rails.logger = proxy_logger
+
+            # We also need to swap some of the Rails railtie loggers.
+            ::ActiveRecord::Base.logger = proxy_logger if defined?(::ActiveRecord::Base)
+            ::ActionController::Base.logger = proxy_logger if defined?(::ActionController::Base)
+            ::ActiveJob::Base.logger = proxy_logger if defined?(::ActiveJob::Base)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/loggers/swaps/scout.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    module Loggers
+      module Swaps
+        # Swaps in our logger for the test Scout logger.
+        class Scout
+          attr_reader :context
+
+          def self.present?
+            defined?(::TestLoggerWrapper) && ::TestLoggerWrapper.logger
+          end
+
+          def initialize(context)
+            @context = context
+          end
+
+          def update_logger!
+            swap_in_proxy_logger!
+
+            new_log_location
+          end
+
+          private
+
+          def log_instance
+            ::TestLoggerWrapper.logger
+          end
+
+          def new_file_logger
+            @new_file_logger ||= Loggers::Logger.new(context, log_instance).create_logger!
+          end
+
+          # Eseentially creates the original logger.
+          def original_logger
+            # We can use the previous logdev. log_device will continuously call write
+            # through the devices until the logdev (@dev) is an IO device other than logdev:
+            # https://github.com/ruby/ruby/blob/master/lib/logger/log_device.rb#L42
+            # Log device holds the configurations around shifting too.
+            original_logdevice = log_instance.instance_variable_get(:@logdev)
+
+            ::Logger.new(original_logdevice).tap do |logger|
+              logger.formatter = log_instance.formatter
+            end
+          end
+
+          def new_log_location
+            new_file_logger.instance_variable_get(:@logdev).filename
+          end
+
+          def swap_in_proxy_logger!
+            # First logger needs to be the original logger for the return value of relayed calls.
+            proxy_logger = Proxy.create_with_loggers(original_logger, new_file_logger)
+
+            ::TestLoggerWrapper.logger = proxy_logger
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/loggers/swaps/sidekiq.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    module Loggers
+      module Swaps
+        # Swaps in our logger for the Sidekiq logger.
+        class Sidekiq
+          attr_reader :context
+
+          def self.present?
+            defined?(::Sidekiq) && ::Sidekiq.logger
+          end
+
+          def initialize(context)
+            @context = context
+          end
+
+          def update_logger!
+            swap_in_proxy_logger!
+
+            new_log_location
+          end
+
+          private
+
+          def log_instance
+            ::Sidekiq.logger
+          end
+
+          def new_file_logger
+            @new_file_logger ||= Loggers::Logger.new(context, log_instance).create_logger!
+          end
+
+          # Eseentially creates the original logger.
+          def original_logger
+            # We can use the previous logdev. log_device will continuously call write
+            # through the devices until the logdev (@dev) is an IO device other than logdev:
+            # https://github.com/ruby/ruby/blob/master/lib/logger/log_device.rb#L42
+            # Log device holds the configurations around shifting too.
+            original_logdevice = log_instance.instance_variable_get(:@logdev)
+
+            ::Logger.new(original_logdevice).tap do |logger|
+              logger.formatter = log_instance.formatter
+            end
+          end
+
+          def new_log_location
+            new_file_logger.instance_variable_get(:@logdev).filename
+          end
+
+          def swap_in_proxy_logger!
+            # First logger needs to be the original logger for the return value of relayed calls.
+            proxy_logger = Proxy.create_with_loggers(original_logger, new_file_logger)
+
+            ::Sidekiq.configure_server do |config|
+              config.logger = proxy_logger
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/monitor/monitor.rb

@@ -29,6 +29,7 @@ module ScoutApm
 
       def initialize
         @context = Context.new
+        context.logger.debug('Monitor instance created')
 
         context.application_root = $stdin.gets&.chomp
 
@@ -83,6 +84,7 @@ module ScoutApm
         exit if fork
         $stdin.reopen '/dev/null'
 
+        context.logger.debug("Monitor process daemonized, PID: #{Process.pid}")
         File.write(context.config.value('monitor_pid_file'), Process.pid)
       end
 

data/lib/scout_apm/logging/monitor_manager/manager.rb

@@ -24,9 +24,6 @@ module ScoutApm
         add_exit_handler!
 
         determine_configuration_state
-
-        # Continue to hold the lock until we have written the PID file.
-        ensure_monitor_pid_file_exists
       end
 
       def determine_configuration_state
@@ -35,6 +32,9 @@ module ScoutApm
         if monitoring_enabled
           context.logger.info('Log monitoring enabled')
           create_process
+
+          # Continue to hold the lock until we have written the PID file.
+          ensure_monitor_pid_file_exists
         else
           context.logger.info('Log monitoring disabled')
           remove_processes
@@ -47,10 +47,16 @@ module ScoutApm
       # treated as the same file as before.
       # If logs get rotated, the fingerprint changes, and the collector automatically detects this.
       def add_exit_handler!
+        # With the use of unicorn and puma worker killer, we want to ensure we only restart (exit and
+        # eventually start) the monitor and collector when the main process exits, and not the workers.
+        initialized_process_id = Process.pid
         at_exit do
           # Only remove/restart the monitor and collector if we are exiting from an app_server process.
           # We need to wait on this check, as the process command line changes at some point.
-          remove_processes if Utils.current_process_is_app_server?
+          if Utils.current_process_is_app_server? && Process.pid == initialized_process_id
+            context.logger.debug('Exiting from app server process. Removing monitor and collector processes.')
+            remove_processes
+          end
         end
       end
 
@@ -64,12 +70,15 @@ module ScoutApm
         gem_directory = File.expand_path('../../../..', __dir__)
 
         # As we daemonize the process, we will write to the pid file within the process.
-        Process.spawn("ruby #{gem_directory}/bin/scout_apm_logging_monitor", in: reader)
+        pid = Process.spawn("ruby #{gem_directory}/bin/scout_apm_logging_monitor", in: reader)
 
         reader.close
         # TODO: Add support for Sinatra.
         writer.puts Rails.root if defined?(Rails)
         writer.close
+        # Block until we have spawned the process and forked. This is to ensure
+        # we keep the exclusive lock until the process has written the PID file.
+        Process.wait(pid)
       end
 
       private
@@ -81,7 +90,10 @@ module ScoutApm
 
         # Naive benchmarks show this taking ~0.01 seconds.
         loop do
-          break if File.exist?(context.config.value('monitor_pid_file'))
+          if File.exist?(context.config.value('monitor_pid_file'))
+            context.logger.debug('Monitor PID file exists. Releasing lock.')
+            break
+          end
 
           if Time.now - start_time > timeout_seconds
             context.logger.warn('Unable to verify monitor PID file write. Releasing lock.')

data/lib/scout_apm/logging/state.rb

@@ -28,18 +28,16 @@ module ScoutApm
               memo[key] = context.config.value(key)
             end
 
-            unless updated_log_locations.empty?
-              contents = file.read
+            contents = file.read
+            old_log_state_files = if contents.empty?
+                                    []
+                                  else
+                                    current_data = JSON.parse(contents)
+                                    current_data['logs_monitored']
+                                  end
 
-              olds_log_files = if contents.empty?
-                                 []
-                               else
-                                 current_data = JSON.parse(contents)
-                                 current_data['logs_monitored']
-                               end
-
-              data['logs_monitored'] = merge_and_dedup_log_locations(updated_log_locations, olds_log_files)
-            end
+            data['logs_monitored'] =
+              merge_and_dedup_log_locations(updated_log_locations, old_log_state_files, data['logs_monitored'])
 
             file.rewind # Move cursor to beginning of the file
             file.truncate(0) # Truncate existing content
@@ -55,9 +53,10 @@ module ScoutApm
 
         # Should we add better detection for similar basenames but different paths?
         # May be a bit tricky with tools like capistrano and releases paths differentiated by time.
-        def merge_and_dedup_log_locations(new_logs, old_logs)
-          # Take the new logs if duplication, as we could be in a newer release.
-          merged = (new_logs + old_logs).each_with_object({}) do |log_path, hash|
+        def merge_and_dedup_log_locations(*log_locations)
+          # Take the new logs if duplication (those first passed in the args) as we could be in a newer release.
+          logs = log_locations.reduce([], :concat)
+          merged = logs.each_with_object({}) do |log_path, hash|
             base_name = File.basename(log_path)
             hash[base_name] ||= log_path
           end

data/lib/scout_apm/logging/version.rb

@@ -2,6 +2,6 @@
 
 module ScoutApm
   module Logging
-    VERSION = '0.0.0.1'
+    VERSION = '0.0.3'
   end
 end

data/lib/scout_apm_logging.rb

@@ -21,7 +21,7 @@ module ScoutApm
         initializer 'scout_apm_logging.monitor' do
           context = ScoutApm::Logging::MonitorManager.instance.context
 
-          Loggers::Capture.new(context).capture_log_locations!
+          Loggers::Capture.new(context).setup!
 
           unless Utils.skip_setup?
             Utils.attempt_exclusive_lock(context) do

data/scout_apm_logging.gemspec

@@ -1,5 +1,3 @@
-require "bundler/gem_tasks"
-
 $LOAD_PATH.push File.expand_path('lib', __dir__)
 require 'scout_apm/logging/version'
 

data/spec/integration/loggers/capture_spec.rb

@@ -2,6 +2,8 @@ require 'logger'
 
 require 'spec_helper'
 
+require_relative '../../../lib/scout_apm/logging/loggers/capture'
+
 describe ScoutApm::Logging::Loggers::Capture do
   it 'should find the logger, capture the log destination, and rotate collector configs' do
     ENV['SCOUT_MONITOR_INTERVAL'] = '10'
@@ -17,9 +19,7 @@ describe ScoutApm::Logging::Loggers::Capture do
     first_logger = ScoutTestLogger.new('/tmp/first_file.log')
     first_logger_basename = File.basename(first_logger.instance_variable_get(:@logdev).filename.to_s)
     first_logger_updated_path = File.join(context.config.value('logs_proxy_log_dir'), first_logger_basename)
-
-    # While we only use the ObjectSpace for the test logger, we need to wait for it to be captured.
-    wait_for_logger
+    TestLoggerWrapper.logger = first_logger
 
     similuate_railtie
 
@@ -36,6 +36,7 @@ describe ScoutApm::Logging::Loggers::Capture do
     second_logger = ScoutTestLogger.new('/tmp/second_file.log')
     second_logger_basename = File.basename(second_logger.instance_variable_get(:@logdev).filename.to_s)
     second_logger_updated_path = File.join(context.config.value('logs_proxy_log_dir'), second_logger_basename)
+    TestLoggerWrapper.logger = second_logger
 
     similuate_railtie
 
@@ -61,18 +62,7 @@ describe ScoutApm::Logging::Loggers::Capture do
   def similuate_railtie
     context = ScoutApm::Logging::MonitorManager.instance.context
 
-    ScoutApm::Logging::Loggers::Capture.new(context).capture_log_locations!
+    ScoutApm::Logging::Loggers::Capture.new(context).setup!
     ScoutApm::Logging::MonitorManager.new.setup!
   end
-
-  def wait_for_logger
-    start_time = Time.now
-    loop do
-      break if ObjectSpace.each_object(ScoutTestLogger).count.positive?
-
-      raise 'Timed out while waiting for logger in ObjectSpace' if Time.now - start_time > 10
-
-      sleep 0.1
-    end
-  end
 end

data/spec/integration/monitor/collector/downloader/will_verify_checksum.rb

@@ -12,7 +12,9 @@ describe ScoutApm::Logging::Collector::Downloader do
 
     File.write(otelcol_contrib_path, 'fake content')
 
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
     ScoutApm::Logging::MonitorManager.instance.setup!
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
 
     # Give the process time to initialize, download the collector, and start it
     wait_for_process_with_timeout!('otelcol-contrib', 20)

data/spec/integration/monitor/collector_healthcheck_spec.rb

@@ -11,7 +11,9 @@ describe ScoutApm::Logging::Monitor do
 
     ScoutApm::Logging::Utils.ensure_directory_exists('/tmp/scout_apm/scout_apm_log_monitor.pid')
 
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
     ScoutApm::Logging::MonitorManager.instance.setup!
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
 
     # Give the process time to initialize, download the collector, and start it
     wait_for_process_with_timeout!('otelcol-contrib', 20)

data/spec/integration/monitor/continuous_state_collector_spec.rb

@@ -11,7 +11,9 @@ describe ScoutApm::Logging::Monitor do
 
     context = ScoutApm::Logging::MonitorManager.instance.context
     collector_pid_location = context.config.value('collector_pid_file')
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
     ScoutApm::Logging::MonitorManager.instance.setup!
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
     # Give the process time to initialize, download the collector, and start it
     wait_for_process_with_timeout!('otelcol-contrib', 20)
 

data/spec/integration/monitor/previous_collector_setup_spec.rb

@@ -11,7 +11,10 @@ describe ScoutApm::Logging::Monitor do
     collector_pid_location = ScoutApm::Logging::MonitorManager.instance.context.config.value('collector_pid_file')
     ScoutApm::Logging::Utils.ensure_directory_exists(monitor_pid_location)
 
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
     ScoutApm::Logging::MonitorManager.instance.setup!
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
+
     # Give the process time to initialize, download the collector, and start it
     wait_for_process_with_timeout!('otelcol-contrib', 20)
 

data/spec/integration/monitor_manager/disable_agent_spec.rb

@@ -9,7 +9,9 @@ describe ScoutApm::Logging::Collector::Manager do
 
     expect(`pgrep otelcol-contrib --runstates D,R,S`).to be_empty
 
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
     ScoutApm::Logging::MonitorManager.instance.setup!
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
 
     wait_for_process_with_timeout!('otelcol-contrib', 20)
 

data/spec/integration/monitor_manager/monitor_pid_file_spec.rb

@@ -15,7 +15,9 @@ describe ScoutApm::Logging::Collector::Manager do
       file.write('123456')
     end
 
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
     ScoutApm::Logging::MonitorManager.instance.setup!
+    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
 
     # Give the process time to initialize, download the collector, and start it
     wait_for_process_with_timeout!('otelcol-contrib', 20)

data/spec/integration/rails/lifecycle_spec.rb

@@ -1,14 +1,17 @@
 require 'spec_helper'
+require_relative '../../rails/app'
 
 describe ScoutApm::Logging do
   it 'checks the Rails lifecycle for creating the daemon and collector processes' do
     ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
 
-    pid_file = ScoutApm::Logging::MonitorManager.instance.context.config.value('monitor_pid_file')
+    context = ScoutApm::Logging::MonitorManager.instance.context
+    pid_file = context.config.value('monitor_pid_file')
     expect(File.exist?(pid_file)).to be_falsey
 
-    make_basic_app
+    rails_pid = fork do
+      initialize_app
+    end
 
     wait_for_process_with_timeout!('otelcol-contrib', 20)
 
@@ -21,6 +24,32 @@ describe ScoutApm::Logging do
     # Check if the process with the stored PID is running
     expect(ScoutApm::Logging::Utils.check_process_liveliness(pid, 'scout_apm_logging_monitor')).to be_truthy
 
+    # Call the app to generate the logs
+    `curl localhost:8080`
+
+    proxy_dir = context.config.value('logs_proxy_log_dir')
+    files = Dir.entries(proxy_dir) - ['.', '..']
+    log_file = File.join(proxy_dir, files[0])
+
+    lines = []
+    File.open(log_file, 'r') do |file|
+      file.each_line do |line|
+        # Parse each line as JSON
+        lines << JSON.parse(line)
+      rescue JSON::ParserError => e
+        puts e
+      end
+    end
+
+    messages = lines.map { |item| item['msg'] }
+
+    # Verify we have all the logs
+    expect(messages.count('[TEST] Some log')).to eq(1)
+    expect(messages.count('[YIELD] Yield Test')).to eq(1)
+    expect(messages.count('Another Log')).to eq(1)
+
+    # Kill the rails process. We use kill as using any other signal throws a long log line.
+    Process.kill('KILL', rails_pid)
     # Kill the process and ensure PID file clean up
     Process.kill('TERM', pid)
     sleep 1 # Give the process time to exit

data/spec/rails/app.rb

@@ -0,0 +1,38 @@
+# https://github.com/rack/rack/pull/1937
+begin
+  require 'rackup'
+rescue LoadError # rubocop:disable Lint/SuppressedException
+end
+
+require 'action_controller/railtie'
+require 'logger'
+require 'scout_apm_logging'
+
+Rails.logger = ActiveSupport::TaggedLogging.new(Logger.new($stdout))
+
+class App < ::Rails::Application
+  config.eager_load = false
+
+  routes.append do
+    root to: 'root#index'
+  end
+end
+
+class RootController < ActionController::Base
+  def index
+    Rails.logger.tagged('TEST').info('Some log')
+    Rails.logger.tagged('YIELD') { logger.info('Yield Test') }
+    Rails.logger.info('Another Log')
+    render plain: Rails.version
+  end
+end
+
+def initialize_app
+  App.initialize!
+
+  if defined?(Rack::Server)
+    Rack::Server.start(app: App)
+  else
+    Rackup::Server.start(app: App)
+  end
+end

data/spec/spec_helper.rb

@@ -21,38 +21,15 @@ class ScoutTestLogger < ::Logger
 end
 
 RSpec.configure do |config|
+  ENV["SCOUT_LOG_FILE_PATH"] = "STDOUT"
   ENV["SCOUT_LOG_LEVEL"] = "debug"
-  ENV["SCOUT_COLLECTOR_LOG_LEVEL"] = "debug"
+  ENV["SCOUT_COLLECTOR_LOG_LEVEL"] = "info"
 
   config.after(:each) do
     RSpec::Mocks.space.reset_all
   end
 end
 
-# Taken from:
-# https://github.com/rails/rails/blob/v7.1.3.2/railties/test/isolation/abstract_unit.rb#L252
-def make_basic_app
-  @app = Class.new(Rails::Application) do
-    def self.name; "RailtiesTestApp"; end
-  end
-  @app.config.hosts << proc { true }
-  @app.config.eager_load = false
-  @app.config.session_store :cookie_store, key: "_myapp_session"
-  @app.config.active_support.deprecation = :log
-  @app.config.log_level = :info
-  @app.config.secret_key_base = "b3c631c314c0bbca50c1b2843150fe33"
-
-  yield @app if block_given?
-  @app.initialize!
-
-  @app.routes.draw do
-    get "/" => "omg#index"
-  end
-
-  require "rack/test"
-  extend ::Rack::Test::Methods
-end
-
 def wait_for_process_with_timeout!(name, timeout_time)
   Timeout::timeout(timeout_time) do
     loop do

data/spec/unit/loggers/capture_spec.rb

@@ -4,7 +4,6 @@ require 'stringio'
 require 'spec_helper'
 
 require_relative '../../../lib/scout_apm/logging/loggers/capture'
-require_relative '../../../lib/scout_apm/logging/loggers/proxy'
 
 def capture_stdout
   old_stdout = $stdout
@@ -29,11 +28,8 @@ describe ScoutApm::Logging::Loggers::Capture do
 
       TestLoggerWrapper.logger = ScoutTestLogger.new($stdout)
 
-      # While we only use the ObjectSpace for the test logger, we need to wait for it to be captured.
-      wait_for_logger
-
       capture = ScoutApm::Logging::Loggers::Capture.new(context)
-      capture.capture_log_locations!
+      capture.setup!
 
       expect(TestLoggerWrapper.logger.class).to eq(ScoutApm::Logging::Loggers::Proxy)
 
@@ -50,15 +46,4 @@ describe ScoutApm::Logging::Loggers::Capture do
 
     expect(output_from_log).to include('TEST')
   end
-
-  def wait_for_logger
-    start_time = Time.now
-    loop do
-      break if ObjectSpace.each_object(::ScoutTestLogger).count.positive?
-
-      raise 'Timed out while waiting for logger in ObjectSpace' if Time.now - start_time > 10
-
-      sleep 0.1
-    end
-  end
 end

data/spec/unit/monitor/collector/configuration_spec.rb

@@ -12,7 +12,7 @@ describe ScoutApm::Logging::Collector::Configuration do
     expect(File.exist?(context.config.value('collector_config_file'))).to be_truthy
     config = YAML.load_file(context.config.value('collector_config_file'))
 
-    expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.telemetryhub.com:4317')
+    expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.scoutotel.com:4317')
     expect(config['exporters']['otlp']['headers']['x-telemetryhub-key']).to eq('00001000010000abc')
     expect(config['receivers']['filelog']['include']).to eq(['/tmp/fake_log_file.log'])
   end
@@ -26,7 +26,7 @@ describe ScoutApm::Logging::Collector::Configuration do
     expect(File.exist?(context.config.value('collector_config_file'))).to be_truthy
     config = YAML.load_file(context.config.value('collector_config_file'))
 
-    expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.telemetryhub.com:4317')
+    expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.scoutotel.com:4317')
     expect(config['exporters']['otlp']['headers']['x-telemetryhub-key']).to eq('00001000010000abc')
     expect(config['receivers']['filelog']['include']).to eq(['/tmp/fake_log_file.log'])
 
@@ -44,7 +44,7 @@ describe ScoutApm::Logging::Collector::Configuration do
     expect(File.exist?(context.config.value('collector_config_file'))).to be_truthy
     config = YAML.load_file(context.config.value('collector_config_file'))
 
-    expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.telemetryhub.com:4317')
+    expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.scoutotel.com:4317')
     expect(config['exporters']['otlp']['headers']['x-telemetryhub-key']).to eq('00001000010000abc')
     expect(config['receivers']['filelog']['include']).to eq(['/tmp/fake_log_file.log'])
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm_logging
 version: !ruby/object:Gem::Version
-  version: 0.0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Scout APM
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-16 00:00:00.000000000 Z
+date: 2024-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout_apm
@@ -76,6 +76,7 @@ files:
 - ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
+- CHANGELOG.md
 - Dockerfile
 - Gemfile
 - README.md
@@ -88,8 +89,11 @@ files:
 - lib/scout_apm/logging/loggers/capture.rb
 - lib/scout_apm/logging/loggers/formatter.rb
 - lib/scout_apm/logging/loggers/logger.rb
+- lib/scout_apm/logging/loggers/patches/tagged_logging.rb
 - lib/scout_apm/logging/loggers/proxy.rb
-- lib/scout_apm/logging/loggers/swap.rb
+- lib/scout_apm/logging/loggers/swaps/rails.rb
+- lib/scout_apm/logging/loggers/swaps/scout.rb
+- lib/scout_apm/logging/loggers/swaps/sidekiq.rb
 - lib/scout_apm/logging/monitor/collector/checksum.rb
 - lib/scout_apm/logging/monitor/collector/configuration.rb
 - lib/scout_apm/logging/monitor/collector/downloader.rb
@@ -116,6 +120,7 @@ files:
 - spec/integration/monitor_manager/monitor_pid_file_spec.rb
 - spec/integration/monitor_manager/single_monitor_spec.rb
 - spec/integration/rails/lifecycle_spec.rb
+- spec/rails/app.rb
 - spec/spec_helper.rb
 - spec/unit/config_spec.rb
 - spec/unit/loggers/capture_spec.rb
@@ -160,6 +165,7 @@ test_files:
 - spec/integration/monitor_manager/monitor_pid_file_spec.rb
 - spec/integration/monitor_manager/single_monitor_spec.rb
 - spec/integration/rails/lifecycle_spec.rb
+- spec/rails/app.rb
 - spec/spec_helper.rb
 - spec/unit/config_spec.rb
 - spec/unit/loggers/capture_spec.rb

data/lib/scout_apm/logging/loggers/swap.rb

@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-require 'logger'
-
-require_relative './formatter'
-require_relative './logger'
-
-module ScoutApm
-  module Logging
-    module Loggers
-      # Swaps in our logger for the application's logger.
-      class Swap
-        attr_reader :context, :log_instance, :new_file_logger
-
-        def initialize(context, log_instance)
-          @context = context
-          @log_instance = log_instance
-        end
-
-        def update_logger!
-          create_proxy_log_dir!
-
-          # In Rails 7.1, broadcast logger was added which allows sinking to multiple IO devices.
-          if defined?(::ActiveSupport::BroadcastLogger) && log_instance.is_a?(::ActiveSupport::BroadcastLogger)
-            add_logger_to_broadcast!
-          else
-            swap_in_proxy_logger!
-          end
-        end
-
-        def log_location
-          new_file_logger.instance_variable_get(:@logdev).filename
-        end
-
-        private
-
-        def add_logger_to_broadcast!
-          @new_file_logger = create_file_logger
-          @new_file_logger.formatter = Loggers::Formatter.new
-
-          log_instance.broadcast_to(new_file_logger)
-        end
-
-        def swap_in_proxy_logger! # rubocop:disable Metrics/AbcSize
-          proxy_logger = Proxy.new
-          # We can use the previous logdev. log_device will continuously call write
-          # through the devices until the logdev (@dev) is an IO device other than logdev:
-          # https://github.com/ruby/ruby/blob/master/lib/logger/log_device.rb#L42
-          # Log device holds the configurations around shifting too.
-          original_logdevice = log_instance.instance_variable_get(:@logdev)
-          updated_original_logger = ::Logger.new(original_logdevice)
-          updated_original_logger.formatter = log_instance.formatter
-
-          @new_file_logger = create_file_logger
-          @new_file_logger.formatter = Loggers::Formatter.new
-
-          # First logger needs to be the original logger for the return value of relayed calls.
-          proxy_logger.add(updated_original_logger)
-          proxy_logger.add(new_file_logger)
-
-          if defined?(::ActiveSupport::Logger) && log_instance.is_a?(::ActiveSupport::Logger)
-            Rails.logger = proxy_logger
-          elsif defined?(::Sidekiq::Logger) && log_instance.is_a?(::Sidekiq::Logger)
-            Sidekiq.configure_server do |config|
-              config.logger = proxy_logger
-            end
-          elsif defined?(::ScoutTestLogger) && log_instance.is_a?(::ScoutTestLogger)
-            TestLoggerWrapper.logger = proxy_logger
-          end
-        end
-
-        def create_file_logger
-          Loggers::Logger.new(context, log_instance).create_logger!
-        end
-
-        def create_proxy_log_dir!
-          Utils.ensure_directory_exists(context.config.value('logs_proxy_log_dir'))
-        end
-      end
-    end
-  end
-end