scout_apm_logging 0.0.0.1

data/lib/scout_apm/logging/monitor/monitor.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+##
+# Launched as a daemon process by the monitor manager at Rails startup.
+##
+require 'json'
+
+require 'scout_apm'
+
+require_relative '../logger'
+require_relative '../context'
+require_relative '../config'
+require_relative '../utils'
+require_relative '../state'
+require_relative './collector/manager'
+
+module ScoutApm
+  module Logging
+    # Entry point for the monitor daemon process.
+    class Monitor
+      attr_reader :context
+      attr_accessor :latest_state_sha
+
+      @@instance = nil
+
+      def self.instance
+        @@instance ||= new
+      end
+
+      def initialize
+        @context = Context.new
+
+        context.application_root = $stdin.gets&.chomp
+
+        # Load in the dynamic and state based config settings.
+        context.config = Config.with_file(context, determine_scout_config_filepath)
+
+        daemonize_process!
+      end
+
+      def setup!
+        context.config.logger.info('Monitor daemon process started')
+
+        add_exit_handler!
+
+        unless has_logs_to_monitor?
+          context.config.logger.warn('No logs are set to be monitored. Please set the `logs_monitored` config setting. Exiting.')
+          return
+        end
+
+        initiate_collector_setup! unless has_previous_collector_setup?
+
+        @latest_state_sha = get_state_file_sha
+
+        run!
+      end
+
+      def run!
+        # Prevent the monitor from checking the collector health before it's fully started.
+        # Having this be configurable is useful for testing.
+        sleep context.config.value('monitor_interval_delay')
+
+        loop do
+          sleep context.config.value('monitor_interval')
+
+          check_collector_health
+
+          check_state_change
+        end
+      end
+
+      # Only useful for testing.
+      def config=(config)
+        context.config = config
+      end
+
+      private
+
+      def daemonize_process!
+        # Similar to that of Process.daemon, but we want to keep the dir, STDOUT and STDERR.
+        exit if fork
+        Process.setsid
+        exit if fork
+        $stdin.reopen '/dev/null'
+
+        File.write(context.config.value('monitor_pid_file'), Process.pid)
+      end
+
+      def has_logs_to_monitor?
+        context.config.value('logs_monitored').any?
+      end
+
+      def has_previous_collector_setup?
+        return false unless context.config.value('health_check_port') != 0
+
+        healthy_response = request_health_check_port("http://localhost:#{context.config.value('health_check_port')}/")
+
+        if healthy_response
+          context.logger.info("Collector already setup on port #{context.config.value('health_check_port')}")
+        else
+          context.logger.info('Setting up new collector')
+        end
+
+        healthy_response
+      end
+
+      def initiate_collector_setup!
+        set_health_check_port!
+
+        Collector::Manager.new(context).setup!
+      end
+
+      def is_port_available?(port)
+        socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+        remote_address = Socket.sockaddr_in(port, '127.0.0.1')
+
+        begin
+          socket.connect_nonblock(remote_address)
+        rescue Errno::EINPROGRESS
+          IO.select(nil, [socket])
+          retry
+        rescue Errno::EISCONN, Errno::ECONNRESET
+          false
+        rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+          true
+        ensure
+          socket.close if socket && !socket.closed?
+        end
+      end
+
+      def set_health_check_port!
+        health_check_port = 13_133
+        until is_port_available?(health_check_port)
+          sleep 0.1
+          health_check_port += 1
+        end
+
+        Config::ConfigDynamic.set_value('health_check_port', health_check_port)
+        context.config.state.flush_state!
+      end
+
+      def request_health_check_port(endpoint)
+        uri = URI(endpoint)
+
+        begin
+          response = Net::HTTP.get_response(uri)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            context.logger.error("Error occurred while checking collector health: #{response.message}")
+            return false
+          end
+        rescue StandardError => e
+          context.logger.error("Error occurred while checking collector health: #{e.message}")
+          return false
+        end
+
+        true
+      end
+
+      def check_collector_health
+        context.logger.debug('Checking collector health')
+        collector_health_endpoint = "http://localhost:#{context.config.value('health_check_port')}/"
+
+        healthy_response = request_health_check_port(collector_health_endpoint)
+
+        initiate_collector_setup! unless healthy_response
+      end
+
+      def remove_collector_process # rubocop:disable Metrics/AbcSize
+        return unless File.exist? context.config.value('collector_pid_file')
+
+        process_id = File.read(context.config.value('collector_pid_file'))
+        return if process_id.empty?
+
+        begin
+          Process.kill('TERM', process_id.to_i)
+        rescue Errno::ENOENT, Errno::ESRCH => e
+          context.logger.error("Error occurred while removing collector process from monitor: #{e.message}")
+        ensure
+          File.delete(context.config.value('collector_pid_file'))
+        end
+      end
+
+      def check_state_change
+        current_sha = get_state_file_sha
+
+        return if current_sha == latest_state_sha
+
+        remove_collector_process
+        initiate_collector_setup!
+
+        # File SHA can change due to port mappings on collector setup.
+        @latest_state_sha = get_state_file_sha
+      end
+
+      def add_exit_handler!
+        at_exit do
+          # There may not be a file to delete, as the monitor manager ensures cleaning it up when monitoring is disabled.
+          File.delete(context.config.value('monitor_pid_file')) if File.exist?(context.config.value('monitor_pid_file'))
+        end
+      end
+
+      def get_state_file_sha
+        return nil unless File.exist?(context.config.value('monitor_state_file'))
+
+        `sha256sum #{context.config.value('monitor_state_file')}`.split(' ').first
+      end
+
+      def determine_scout_config_filepath
+        "#{context.application_root}/config/scout_apm.yml"
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/monitor_manager/manager.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    # Manages the creation of the daemon monitor process.
+    class MonitorManager
+      attr_reader :context
+
+      @@instance = nil
+
+      def self.instance
+        @@instance ||= new
+      end
+
+      def initialize
+        @context = Context.new
+        context.config = Config.with_file(context, context.config.value('config_file'))
+      end
+
+      def setup!
+        context.config.log_settings(context.logger)
+        context.logger.info('Setting up monitor daemon process')
+
+        add_exit_handler!
+
+        determine_configuration_state
+
+        # Continue to hold the lock until we have written the PID file.
+        ensure_monitor_pid_file_exists
+      end
+
+      def determine_configuration_state
+        monitoring_enabled = context.config.value('logs_monitor')
+
+        if monitoring_enabled
+          context.logger.info('Log monitoring enabled')
+          create_process
+        else
+          context.logger.info('Log monitoring disabled')
+          remove_processes
+        end
+      end
+
+      # With the use of fileoffsets in the collector, and the persistent queue of already collected logs,
+      # we can safely restart the collector. Due to the way fingerprinting of the files works, if the
+      # file path switches, but the beginning contents of the file remain the same, the file will be
+      # treated as the same file as before.
+      # If logs get rotated, the fingerprint changes, and the collector automatically detects this.
+      def add_exit_handler!
+        at_exit do
+          # Only remove/restart the monitor and collector if we are exiting from an app_server process.
+          # We need to wait on this check, as the process command line changes at some point.
+          remove_processes if Utils.current_process_is_app_server?
+        end
+      end
+
+      def create_process
+        return if process_exists?
+
+        Utils.ensure_directory_exists(context.config.value('monitor_pid_file'))
+
+        reader, writer = IO.pipe
+
+        gem_directory = File.expand_path('../../../..', __dir__)
+
+        # As we daemonize the process, we will write to the pid file within the process.
+        Process.spawn("ruby #{gem_directory}/bin/scout_apm_logging_monitor", in: reader)
+
+        reader.close
+        # TODO: Add support for Sinatra.
+        writer.puts Rails.root if defined?(Rails)
+        writer.close
+      end
+
+      private
+
+      def ensure_monitor_pid_file_exists
+        start_time = Time.now
+        # We don't want to hold up the initial Rails boot time for very long.
+        timeout_seconds = 0.1
+
+        # Naive benchmarks show this taking ~0.01 seconds.
+        loop do
+          break if File.exist?(context.config.value('monitor_pid_file'))
+
+          if Time.now - start_time > timeout_seconds
+            context.logger.warn('Unable to verify monitor PID file write. Releasing lock.')
+            break
+          end
+
+          sleep 0.01
+        end
+      end
+
+      def process_exists?
+        return false unless File.exist? context.config.value('monitor_pid_file')
+
+        process_id = File.read(context.config.value('monitor_pid_file'))
+        return false if process_id.empty?
+
+        process_exists = Utils.check_process_liveliness(process_id.to_i, 'scout_apm_logging_monitor')
+        File.delete(context.config.value('monitor_pid_file')) unless process_exists
+
+        process_exists
+      end
+
+      def remove_monitor_process # rubocop:disable Metrics/AbcSize
+        return unless File.exist? context.config.value('monitor_pid_file')
+
+        process_id = File.read(context.config.value('monitor_pid_file'))
+        return if process_id.empty?
+
+        begin
+          Process.kill('TERM', process_id.to_i)
+        rescue Errno::ENOENT, Errno::ESRCH => e
+          context.logger.error("Error occurred while removing monitor process: #{e.message}")
+          File.delete(context.config.value('monitor_pid_file'))
+        end
+      end
+
+      def remove_collector_process # rubocop:disable Metrics/AbcSize
+        return unless File.exist? context.config.value('collector_pid_file')
+
+        process_id = File.read(context.config.value('collector_pid_file'))
+        return if process_id.empty?
+
+        begin
+          Process.kill('TERM', process_id.to_i)
+        rescue Errno::ENOENT, Errno::ESRCH => e
+          context.logger.error("Error occurred while removing collector process from manager: #{e.message}")
+        ensure
+          File.delete(context.config.value('collector_pid_file'))
+        end
+      end
+
+      def remove_data_file
+        return unless File.exist? context.config.value('monitor_state_file')
+
+        File.delete(context.config.value('monitor_state_file'))
+      end
+
+      # Remove both the monitor and collector processes that we have spawned.
+      def remove_processes
+        remove_monitor_process
+        remove_collector_process
+        remove_data_file
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/state.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    class Config
+      # Responsibling for ensuring safe interprocess persitance around configuration state.
+      class State
+        attr_reader :context
+
+        def initialize(context)
+          @context = context
+        end
+
+        def load_state_from_file
+          return unless File.exist?(context.config.value('monitor_state_file'))
+
+          file_contents = File.read(context.config.value('monitor_state_file'))
+          JSON.parse(file_contents)
+        end
+
+        def flush_to_file!(updated_log_locations = []) # rubocop:disable Metrics/AbcSize
+          Utils.ensure_directory_exists(context.config.value('monitor_state_file'))
+
+          File.open(context.config.value('monitor_state_file'), (File::RDWR | File::CREAT), 0o644) do |file|
+            file.flock(File::LOCK_EX)
+
+            data = Config::ConfigState.get_values_to_set.each_with_object({}) do |key, memo|
+              memo[key] = context.config.value(key)
+            end
+
+            unless updated_log_locations.empty?
+              contents = file.read
+
+              olds_log_files = if contents.empty?
+                                 []
+                               else
+                                 current_data = JSON.parse(contents)
+                                 current_data['logs_monitored']
+                               end
+
+              data['logs_monitored'] = merge_and_dedup_log_locations(updated_log_locations, olds_log_files)
+            end
+
+            file.rewind # Move cursor to beginning of the file
+            file.truncate(0) # Truncate existing content
+            file.write(JSON.pretty_generate(data))
+          rescue StandardError => e
+            context.logger.error("Error occurred while flushing state to file: #{e.message}. Unlocking.")
+          ensure
+            file.flock(File::LOCK_UN)
+          end
+        end
+
+        private
+
+        # Should we add better detection for similar basenames but different paths?
+        # May be a bit tricky with tools like capistrano and releases paths differentiated by time.
+        def merge_and_dedup_log_locations(new_logs, old_logs)
+          # Take the new logs if duplication, as we could be in a newer release.
+          merged = (new_logs + old_logs).each_with_object({}) do |log_path, hash|
+            base_name = File.basename(log_path)
+            hash[base_name] ||= log_path
+          end
+
+          merged.values
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/utils.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module ScoutApm
+  module Logging
+    # Miscellaneous utilities for the logging module.
+    module Utils
+      # Takes a complete file path, and ensures that the directory structure exists.
+      def self.ensure_directory_exists(file_path)
+        file_path = File.dirname(file_path) unless file_path[-1] == '/'
+
+        FileUtils.mkdir_p(file_path) unless File.directory?(file_path)
+      end
+
+      # TODO: Add support for other platforms
+      def self.get_architecture
+        if /arm/ =~ RbConfig::CONFIG['arch']
+          'arm64'
+        else
+          'amd64'
+        end
+      end
+
+      def self.get_host_os
+        if /darwin|mac os/ =~ RbConfig::CONFIG['host_os']
+          'darwin'
+        else
+          'linux'
+        end
+      end
+
+      def self.check_process_liveliness(pid, name)
+        # Pipe to cat to prevent truncation of the output
+        process_information = `ps -p #{pid} -o pid=,stat=,command= | cat`
+        return false if process_information.empty?
+
+        process_information_parts = process_information.split(' ')
+        process_information_status = process_information_parts[1]
+
+        return false if process_information_status == 'Z'
+        return false unless process_information.include?(name)
+
+        true
+      end
+
+      def self.current_process_is_app_server?
+        # TODO: Add more app servers.
+        process_command = `ps -p #{Process.pid} -o command= | cat`.downcase
+        [
+          process_command.include?('puma'),
+          process_command.include?('unicorn'),
+          process_command.include?('passenger')
+        ].any?
+      end
+
+      def self.skip_setup?
+        [
+          ARGV.include?('assets:precompile'),
+          ARGV.include?('assets:clean'),
+          (defined?(::Rails::Console) && $stdout.isatty && $stdin.isatty)
+        ].any?
+      end
+
+      def self.attempt_exclusive_lock(context)
+        lock_file = context.config.value('manager_lock_file')
+        ensure_directory_exists(lock_file)
+
+        begin
+          file = File.open(lock_file, File::RDWR | File::CREAT | File::EXCL)
+        rescue Errno::EEXIST
+          context.logger.info('Manager lock file held, continuing.')
+          return
+        end
+
+        # Ensure the lock file is deleted when the block completes
+        begin
+          yield
+        ensure
+          file.close
+          File.delete(lock_file) if File.exist?(lock_file)
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    VERSION = '0.0.0.1'
+  end
+end

data/lib/scout_apm_logging.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'scout_apm'
+
+require 'scout_apm/logging/config'
+require 'scout_apm/logging/logger'
+require 'scout_apm/logging/context'
+require 'scout_apm/logging/utils'
+require 'scout_apm/logging/state'
+
+require 'scout_apm/logging/loggers/capture'
+
+require 'scout_apm/logging/monitor_manager/manager'
+
+module ScoutApm
+  ## This module is responsible for setting up monitoring of the application's logs.
+  module Logging
+    if defined?(Rails) && defined?(Rails::Railtie)
+      # If we are in a Rails environment, setup the monitor daemon manager.
+      class RailTie < ::Rails::Railtie
+        initializer 'scout_apm_logging.monitor' do
+          context = ScoutApm::Logging::MonitorManager.instance.context
+
+          Loggers::Capture.new(context).capture_log_locations!
+
+          unless Utils.skip_setup?
+            Utils.attempt_exclusive_lock(context) do
+              ScoutApm::Logging::MonitorManager.instance.setup!
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/scout_apm_logging.gemspec

@@ -0,0 +1,27 @@
+require "bundler/gem_tasks"
+
+$LOAD_PATH.push File.expand_path('lib', __dir__)
+require 'scout_apm/logging/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'scout_apm_logging'
+  s.version     = ScoutApm::Logging::VERSION
+  s.authors     = 'Scout APM'
+  s.email       = ['support@scoutapp.com']
+  s.homepage    = 'https://github.com/scoutapp/scout_apm_ruby_logging'
+  s.summary     = 'Ruby Logging Support'
+  s.description = 'Sets up log monitoring for Scout APM Ruby clients.'
+  s.license     = 'MIT'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ['lib']
+
+  s.required_ruby_version = '>= 2.6'
+
+  s.add_dependency 'scout_apm'
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'rubocop', '1.50.2'
+  s.add_development_dependency 'rubocop-ast', '1.30.0'
+end

data/spec/data/config_test_1.yml

@@ -0,0 +1,27 @@
+common: &defaults
+  name: Scout APM
+  key: 000011110000
+  log_level: debug
+  monitor: true
+  
+  ###
+  # Logging
+  ###
+  logs_monitor: true
+  logs_ingest_key: "00001000010000abc"
+  logs_monitored: ["/tmp/fake_log_file.log"]
+
+production:
+  <<: *defaults
+  name: APM Test Conf (Production)
+
+development:
+  <<: *defaults
+  name: APM Test Conf (Development)
+  host: http://localhost:3000
+  monitor: true
+
+test:
+  <<: *defaults
+  name: APM Test Conf (Test)
+  monitor: false

data/spec/data/logs_config.yml

@@ -0,0 +1,3 @@
+extensions:
+  file_storage/otc:
+    directory: /dev/null

data/spec/data/mock_config.yml

@@ -0,0 +1,29 @@
+common: &defaults
+  name: Scout APM
+  key: 000011110000
+  log_level: debug
+  monitor: true
+  
+  ###
+  # Logging
+  ###
+  logs_monitor: true
+  logs_ingest_key: "00001000010000abc"
+  logs_monitored: ["/tmp/fake_log_file.log"]
+  # Need to give a high enough number for the original health check to pass
+  monitor_interval: 10
+
+production:
+  <<: *defaults
+  name: APM Test Conf (Production)
+
+development:
+  <<: *defaults
+  name: APM Test Conf (Development)
+  host: http://localhost:3000
+  monitor: true
+
+test:
+  <<: *defaults
+  name: APM Test Conf (Test)
+  monitor: false

data/spec/data/state_file.json

@@ -0,0 +1,3 @@
+{
+  "health_check_port": 1234
+}