scout_apm_logging 0.0.13 → 1.0.1

data/lib/scout_apm/logging/monitor/monitor.rb

@@ -1,216 +0,0 @@
-# frozen_string_literal: true
-
-##
-# Launched as a daemon process by the monitor manager at Rails startup.
-##
-require 'json'
-
-require 'scout_apm'
-
-require_relative '../logger'
-require_relative '../context'
-require_relative '../config'
-require_relative '../utils'
-require_relative '../state'
-require_relative './collector/manager'
-
-module ScoutApm
-  module Logging
-    # Entry point for the monitor daemon process.
-    class Monitor
-      attr_reader :context
-      attr_accessor :latest_state_sha
-
-      @@instance = nil
-
-      def self.instance
-        @@instance ||= new
-      end
-
-      def initialize
-        @context = Context.new
-        context.logger.debug('Monitor instance created')
-
-        context.application_root = $stdin.gets&.chomp
-
-        # Load in the dynamic and state based config settings.
-        context.config = Config.with_file(context, determine_scout_config_filepath)
-
-        daemonize_process!
-      end
-
-      def setup!
-        context.config.logger.info('Monitor daemon process started')
-
-        add_exit_handler!
-
-        unless has_logs_to_monitor?
-          context.config.logger.warn('No logs are set to be monitored. Please set the `logs_monitored` config setting. Exiting.')
-          return
-        end
-
-        initiate_collector_setup! unless has_previous_collector_setup?
-
-        @latest_state_sha = get_state_file_sha
-
-        run!
-      end
-
-      def run!
-        # Prevent the monitor from checking the collector health before it's fully started.
-        # Having this be configurable is useful for testing.
-        sleep context.config.value('monitor_interval_delay')
-
-        loop do
-          sleep context.config.value('monitor_interval')
-
-          check_collector_health
-
-          check_state_change
-        end
-      end
-
-      # Only useful for testing.
-      def config=(config)
-        context.config = config
-      end
-
-      private
-
-      def daemonize_process!
-        # Similar to that of Process.daemon, but we want to keep the dir, STDOUT and STDERR.
-        exit if fork
-        Process.setsid
-        exit if fork
-        $stdin.reopen '/dev/null'
-
-        context.logger.debug("Monitor process daemonized, PID: #{Process.pid}")
-        File.write(context.config.value('monitor_pid_file'), Process.pid)
-      end
-
-      def has_logs_to_monitor?
-        context.config.value('logs_monitored').any?
-      end
-
-      def has_previous_collector_setup?
-        return false unless context.config.value('health_check_port') != 0
-
-        healthy_response = request_health_check_port("http://localhost:#{context.config.value('health_check_port')}/")
-
-        if healthy_response
-          context.logger.info("Collector already setup on port #{context.config.value('health_check_port')}")
-        else
-          context.logger.info('Setting up new collector')
-        end
-
-        healthy_response
-      end
-
-      def initiate_collector_setup!
-        set_health_check_port!
-
-        Collector::Manager.new(context).setup!
-      end
-
-      def is_port_available?(port)
-        socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
-        remote_address = Socket.sockaddr_in(port, '127.0.0.1')
-
-        begin
-          socket.connect_nonblock(remote_address)
-        rescue Errno::EINPROGRESS
-          IO.select(nil, [socket])
-          retry
-        rescue Errno::EISCONN, Errno::ECONNRESET
-          false
-        rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
-          true
-        ensure
-          socket.close if socket && !socket.closed?
-        end
-      end
-
-      def set_health_check_port!
-        health_check_port = 13_133
-        until is_port_available?(health_check_port)
-          sleep 0.1
-          health_check_port += 1
-        end
-
-        Config::ConfigDynamic.set_value('health_check_port', health_check_port)
-        context.config.state.flush_state!
-      end
-
-      def request_health_check_port(endpoint)
-        uri = URI(endpoint)
-
-        begin
-          response = Net::HTTP.get_response(uri)
-
-          unless response.is_a?(Net::HTTPSuccess)
-            context.logger.error("Error occurred while checking collector health: #{response.message}")
-            return false
-          end
-        rescue StandardError => e
-          context.logger.error("Error occurred while checking collector health: #{e.message}")
-          return false
-        end
-
-        true
-      end
-
-      def check_collector_health
-        context.logger.debug('Checking collector health')
-        collector_health_endpoint = "http://localhost:#{context.config.value('health_check_port')}/"
-
-        healthy_response = request_health_check_port(collector_health_endpoint)
-
-        initiate_collector_setup! unless healthy_response
-      end
-
-      def remove_collector_process # rubocop:disable Metrics/AbcSize
-        return unless File.exist? context.config.value('collector_pid_file')
-
-        process_id = File.read(context.config.value('collector_pid_file'))
-        return if process_id.empty?
-
-        begin
-          Process.kill('TERM', process_id.to_i)
-        rescue Errno::ENOENT, Errno::ESRCH => e
-          context.logger.error("Error occurred while removing collector process from monitor: #{e.message}")
-        ensure
-          File.delete(context.config.value('collector_pid_file'))
-        end
-      end
-
-      def check_state_change
-        current_sha = get_state_file_sha
-
-        return if current_sha == latest_state_sha
-
-        remove_collector_process
-        initiate_collector_setup!
-
-        # File SHA can change due to port mappings on collector setup.
-        @latest_state_sha = get_state_file_sha
-      end
-
-      def add_exit_handler!
-        at_exit do
-          # There may not be a file to delete, as the monitor manager ensures cleaning it up when monitoring is disabled.
-          File.delete(context.config.value('monitor_pid_file')) if File.exist?(context.config.value('monitor_pid_file'))
-        end
-      end
-
-      def get_state_file_sha
-        return nil unless File.exist?(context.config.value('monitor_state_file'))
-
-        `sha256sum #{context.config.value('monitor_state_file')}`.split(' ').first
-      end
-
-      def determine_scout_config_filepath
-        "#{context.application_root}/config/scout_apm.yml"
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor_manager/manager.rb

@@ -1,162 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    # Manages the creation of the daemon monitor process.
-    class MonitorManager
-      attr_reader :context
-
-      @@instance = nil
-
-      def self.instance
-        @@instance ||= new
-      end
-
-      def initialize
-        @context = Context.new
-        context.config = Config.with_file(context, context.config.value('config_file'))
-      end
-
-      def setup!
-        context.config.log_settings(context.logger)
-        context.logger.info('Setting up monitor daemon process')
-
-        add_exit_handler!
-
-        determine_configuration_state
-      end
-
-      def determine_configuration_state
-        monitoring_enabled = context.config.value('logs_monitor')
-
-        if monitoring_enabled
-          context.logger.info('Log monitoring enabled')
-          create_process
-
-          # Continue to hold the lock until we have written the PID file.
-          ensure_monitor_pid_file_exists
-        else
-          context.logger.info('Log monitoring disabled')
-          remove_processes
-        end
-      end
-
-      # With the use of fileoffsets in the collector, and the persistent queue of already collected logs,
-      # we can safely restart the collector. Due to the way fingerprinting of the files works, if the
-      # file path switches, but the beginning contents of the file remain the same, the file will be
-      # treated as the same file as before.
-      # If logs get rotated, the fingerprint changes, and the collector automatically detects this.
-      def add_exit_handler!
-        # With the use of unicorn and puma worker killer, we want to ensure we only restart (exit and
-        # eventually start) the monitor and collector when the main process exits, and not the workers.
-        initialized_process_id = Process.pid
-        at_exit do
-          # Only remove/restart the monitor and collector if we are exiting from an app_server process.
-          # We need to wait on this check, as the process command line changes at some point.
-          if Utils.current_process_is_app_server? && Process.pid == initialized_process_id
-            context.logger.debug('Exiting from app server process. Removing monitor and collector processes.')
-            remove_processes
-          end
-        end
-      end
-
-      def create_process
-        return if process_exists?
-
-        Utils.ensure_directory_exists(context.config.value('monitor_pid_file'))
-
-        reader, writer = IO.pipe
-
-        gem_directory = File.expand_path('../../../..', __dir__)
-
-        # As we daemonize the process, we will write to the pid file within the process.
-        pid = Process.spawn("ruby #{gem_directory}/bin/scout_apm_logging_monitor", in: reader)
-
-        reader.close
-        # TODO: Add support for Sinatra.
-        writer.puts Rails.root if defined?(Rails)
-        writer.close
-        # Block until we have spawned the process and forked. This is to ensure
-        # we keep the exclusive lock until the process has written the PID file.
-        Process.wait(pid)
-      end
-
-      private
-
-      def ensure_monitor_pid_file_exists
-        start_time = Time.now
-        # We don't want to hold up the initial Rails boot time for very long.
-        timeout_seconds = 0.1
-
-        # Naive benchmarks show this taking ~0.01 seconds.
-        loop do
-          if File.exist?(context.config.value('monitor_pid_file'))
-            context.logger.debug('Monitor PID file exists. Releasing lock.')
-            break
-          end
-
-          if Time.now - start_time > timeout_seconds
-            context.logger.warn('Unable to verify monitor PID file write. Releasing lock.')
-            break
-          end
-
-          sleep 0.01
-        end
-      end
-
-      def process_exists?
-        return false unless File.exist? context.config.value('monitor_pid_file')
-
-        process_id = File.read(context.config.value('monitor_pid_file'))
-        return false if process_id.empty?
-
-        process_exists = Utils.check_process_liveliness(process_id.to_i, 'scout_apm_logging_monitor')
-        File.delete(context.config.value('monitor_pid_file')) unless process_exists
-
-        process_exists
-      end
-
-      def remove_monitor_process # rubocop:disable Metrics/AbcSize
-        return unless File.exist? context.config.value('monitor_pid_file')
-
-        process_id = File.read(context.config.value('monitor_pid_file'))
-        return if process_id.empty?
-
-        begin
-          Process.kill('TERM', process_id.to_i)
-        rescue Errno::ENOENT, Errno::ESRCH => e
-          context.logger.error("Error occurred while removing monitor process: #{e.message}")
-          File.delete(context.config.value('monitor_pid_file'))
-        end
-      end
-
-      def remove_collector_process # rubocop:disable Metrics/AbcSize
-        return unless File.exist? context.config.value('collector_pid_file')
-
-        process_id = File.read(context.config.value('collector_pid_file'))
-        return if process_id.empty?
-
-        begin
-          Process.kill('TERM', process_id.to_i)
-        rescue Errno::ENOENT, Errno::ESRCH => e
-          context.logger.error("Error occurred while removing collector process from manager: #{e.message}")
-        ensure
-          File.delete(context.config.value('collector_pid_file'))
-        end
-      end
-
-      def remove_data_file
-        return unless File.exist? context.config.value('monitor_state_file')
-
-        File.delete(context.config.value('monitor_state_file'))
-      end
-
-      # Remove both the monitor and collector processes that we have spawned.
-      def remove_processes
-        remove_monitor_process
-        remove_collector_process
-        remove_data_file
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/state.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    class Config
-      # Responsibling for ensuring safe interprocess persitance around configuration state.
-      class State
-        attr_reader :context
-
-        def initialize(context)
-          @context = context
-        end
-
-        def load_state_from_file
-          return unless File.exist?(context.config.value('monitor_state_file'))
-
-          file_contents = File.read(context.config.value('monitor_state_file'))
-          JSON.parse(file_contents)
-        end
-
-        def flush_to_file!(updated_log_locations = []) # rubocop:disable Metrics/AbcSize
-          Utils.ensure_directory_exists(context.config.value('monitor_state_file'))
-
-          File.open(context.config.value('monitor_state_file'), (File::RDWR | File::CREAT), 0o644) do |file|
-            file.flock(File::LOCK_EX)
-
-            data = Config::ConfigState.get_values_to_set.each_with_object({}) do |key, memo|
-              memo[key] = context.config.value(key)
-            end
-
-            contents = file.read
-            old_log_state_files = if contents.empty?
-                                    []
-                                  else
-                                    current_data = JSON.parse(contents)
-                                    current_data['logs_monitored']
-                                  end
-
-            data['logs_monitored'] =
-              merge_and_dedup_log_locations(updated_log_locations, old_log_state_files, data['logs_monitored'])
-
-            file.rewind # Move cursor to beginning of the file
-            file.truncate(0) # Truncate existing content
-            file.write(JSON.pretty_generate(data))
-          rescue StandardError => e
-            context.logger.error("Error occurred while flushing state to file: #{e.message}. Unlocking.")
-          ensure
-            file.flock(File::LOCK_UN)
-          end
-        end
-
-        private
-
-        # Should we add better detection for similar basenames but different paths?
-        # May be a bit tricky with tools like capistrano and releases paths differentiated by time.
-        def merge_and_dedup_log_locations(*log_locations)
-          # Take the new logs if duplication (those first passed in the args) as we could be in a newer release.
-          logs = log_locations.reduce([], :concat)
-          merged = logs.each_with_object({}) do |log_path, hash|
-            base_name = File.basename(log_path)
-            hash[base_name] ||= log_path
-          end
-
-          merged.values
-        end
-      end
-    end
-  end
-end

data/spec/data/logs_config.yml

@@ -1,3 +0,0 @@
-extensions:
-  file_storage/otc:
-    directory: /dev/null

data/spec/data/state_file.json

@@ -1,3 +0,0 @@
-{
-  "health_check_port": 1234
-}

data/spec/integration/loggers/capture_spec.rb

@@ -1,68 +0,0 @@
-require 'logger'
-
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/loggers/capture'
-
-describe ScoutApm::Logging::Loggers::Capture do
-  it 'should find the logger, capture the log destination, and rotate collector configs' do
-    ENV['SCOUT_MONITOR_INTERVAL'] = '10'
-    ENV['SCOUT_MONITOR_INTERVAL_DELAY'] = '10'
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-
-    context = ScoutApm::Logging::MonitorManager.instance.context
-
-    state_file_location = context.config.value('monitor_state_file')
-    collector_pid_location = context.config.value('collector_pid_file')
-    ScoutApm::Logging::Utils.ensure_directory_exists(state_file_location)
-
-    first_logger = ScoutTestLogger.new('/tmp/first_file.log')
-    first_logger_basename = File.basename(first_logger.instance_variable_get(:@logdev).filename.to_s)
-    first_logger_updated_path = File.join(context.config.value('logs_proxy_log_dir'), first_logger_basename)
-    TestLoggerWrapper.logger = first_logger
-
-    similuate_railtie
-
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-    collector_pid = File.read(collector_pid_location)
-
-    content = File.read(state_file_location)
-    data = JSON.parse(content)
-    expect(data['logs_monitored']).to eq([first_logger_updated_path])
-
-    second_logger = ScoutTestLogger.new('/tmp/second_file.log')
-    second_logger_basename = File.basename(second_logger.instance_variable_get(:@logdev).filename.to_s)
-    second_logger_updated_path = File.join(context.config.value('logs_proxy_log_dir'), second_logger_basename)
-    TestLoggerWrapper.logger = second_logger
-
-    similuate_railtie
-
-    content = File.read(state_file_location)
-    data = JSON.parse(content)
-
-    expect(data['logs_monitored'].sort).to eq([first_logger_updated_path, second_logger_updated_path])
-
-    # Need to wait for the delay first health check, next monitor interval to restart the collector, and then for
-    # the collector to restart
-    sleep 25
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-    new_collector_pid = File.read(collector_pid_location)
-
-    # Should have restarted the collector based on the change
-    expect(new_collector_pid).not_to eq(collector_pid)
-  end
-
-  private
-
-  def similuate_railtie
-    context = ScoutApm::Logging::MonitorManager.instance.context
-
-    ScoutApm::Logging::Loggers::Capture.new(context).setup!
-    ScoutApm::Logging::MonitorManager.new.setup!
-  end
-end

data/spec/integration/monitor/collector/downloader/will_verify_checksum.rb

@@ -1,49 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../../../lib/scout_apm/logging/monitor/collector/downloader'
-
-describe ScoutApm::Logging::Collector::Downloader do
-  it 'should validate checksum, and correct download if neccessary' do
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    otelcol_contrib_path = '/tmp/scout_apm/otelcol-contrib'
-    ScoutApm::Logging::Utils.ensure_directory_exists(otelcol_contrib_path)
-
-    File.write(otelcol_contrib_path, 'fake content')
-
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    download_time = File.mtime(otelcol_contrib_path)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-
-    ENV['SCOUT_LOGS_MONITOR'] = 'false'
-
-    ScoutApm::Logging::MonitorManager.new.setup!
-
-    sleep 5 # Give the process time to exit
-
-    expect(File.exist?(ScoutApm::Logging::MonitorManager.instance.context.config.value('monitor_pid_file'))).to be_falsey
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).to be_empty
-    expect(`pgrep scout_apm_log_monitor --runstates D,R,S`).to be_empty
-
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-
-    ScoutApm::Logging::MonitorManager.new.setup!
-
-    # Give the process time to exit, and for the healthcheck to restart it
-    wait_for_process_with_timeout!('otelcol-contrib', 30)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-
-    recheck_time = File.mtime(otelcol_contrib_path)
-
-    expect(download_time).to eq(recheck_time)
-  end
-end

data/spec/integration/monitor/collector_healthcheck_spec.rb

@@ -1,29 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/monitor/monitor'
-
-describe ScoutApm::Logging::Monitor do
-  it 'should recreate collector process on healthcheck if it has exited' do
-    ENV['SCOUT_MONITOR_INTERVAL'] = '10'
-    ENV['SCOUT_MONITOR_INTERVAL_DELAY'] = '10'
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    ScoutApm::Logging::Utils.ensure_directory_exists('/tmp/scout_apm/scout_apm_log_monitor.pid')
-
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-
-    # Bypass gracefull shutdown
-    `pkill -9 otelcol-contrib`
-
-    # Give the process time to exit, and for the healthcheck to restart it
-    wait_for_process_with_timeout!('otelcol-contrib', 30)
-  end
-end

data/spec/integration/monitor/continuous_state_collector_spec.rb

@@ -1,31 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/monitor/monitor'
-
-describe ScoutApm::Logging::Monitor do
-  it "Should not restart the collector if the state hasn't changed" do
-    ENV['SCOUT_MONITOR_INTERVAL'] = '10'
-    ENV['SCOUT_MONITOR_INTERVAL_DELAY'] = '10'
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    context = ScoutApm::Logging::MonitorManager.instance.context
-    collector_pid_location = context.config.value('collector_pid_file')
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-    collector_pid = File.read(collector_pid_location)
-
-    # Give time for the monitor interval to run.
-    sleep 30
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-    second_read_pid = File.read(collector_pid_location)
-
-    expect(second_read_pid).to eq(collector_pid)
-  end
-end

data/spec/integration/monitor/previous_collector_setup_spec.rb

@@ -1,45 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/monitor/monitor'
-
-describe ScoutApm::Logging::Monitor do
-  it 'should use previous collector setup if monitor daemon exits' do
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    monitor_pid_location = ScoutApm::Logging::MonitorManager.instance.context.config.value('monitor_pid_file')
-    collector_pid_location = ScoutApm::Logging::MonitorManager.instance.context.config.value('collector_pid_file')
-    ScoutApm::Logging::Utils.ensure_directory_exists(monitor_pid_location)
-
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    monitor_pid = File.read(monitor_pid_location)
-
-    otelcol_pid = `pgrep otelcol-contrib --runstates D,R,S`.strip!
-    stored_otelcol_pid = File.read(collector_pid_location)
-    expect(otelcol_pid).to eq(stored_otelcol_pid)
-
-    `kill -9 #{monitor_pid}`
-
-    # Create a separate monitor manager instance, or else we won't reload
-    # the configuraiton state.
-    ScoutApm::Logging::MonitorManager.new.setup!
-
-    sleep 5
-
-    expect(`pgrep -f /app/bin/scout_apm_logging_monitor --runstates D,R,S`).not_to be_empty
-
-    new_monitor_pid = File.read(monitor_pid_location)
-    expect(new_monitor_pid).not_to eq(monitor_pid)
-
-    should_be_same_otelcol_pid = `pgrep otelcol-contrib --runstates D,R,S`.strip!
-    should_be_same_stored_otelcol_pid = File.read(collector_pid_location)
-    expect(should_be_same_otelcol_pid).to eq(otelcol_pid)
-    expect(should_be_same_stored_otelcol_pid).to eq(stored_otelcol_pid)
-  end
-end