scout_apm_logging 0.0.12 → 1.0.0

data/lib/scout_apm/logging/monitor/collector/downloader.rb

@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    module Collector
-      # Downloads the collector-contrib binary from the OpenTelemetry project.
-      class Downloader
-        attr_accessor :failed_count
-        attr_reader :context, :checksum
-
-        def initialize(context)
-          @context = context
-          @checksum = Checksum.new(context)
-        end
-
-        def download!
-          # Already downloaded the collector. Noop.
-          return if checksum.verified_checksum?
-
-          # Account for issues such as failed extractions or download corruptions.
-          download_collector
-          verify_checksum
-        rescue StandardError => e
-          # Bypass Rubcop useless asignment rule.
-          failed_count ||= 0
-
-          if failed_count < 3
-            context.logger.error("Failed to download or extract otelcol-contrib: #{e}. Retrying...")
-            failed_count += 1
-            retry
-          end
-        end
-
-        def download_collector(url = nil, redirect: false) # rubocop:disable Metrics/AbcSize
-          # Prevent double logging.
-          unless redirect
-            context.logger.debug("Downloading otelcol-contrib for version #{context.config.value('collector_version')}")
-          end
-
-          url_to_download = url || collector_url
-          uri = URI(url_to_download)
-
-          Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
-            request = Net::HTTP::Get.new(uri)
-            http.request(request) do |response|
-              return download_collector(response['location'], redirect: true) if response.code == '302'
-
-              File.open(destination, 'wb') do |file|
-                response.read_body do |chunk|
-                  file.write(chunk)
-                end
-              end
-            end
-          end
-        end
-
-        private
-
-        def verify_checksum
-          raise 'Invalid checksum on download.' unless checksum.verified_checksum?
-        end
-
-        def collector_url
-          collector_version = context.config.value('collector_version')
-          architecture = Utils.get_architecture
-          host_os = Utils.get_host_os
-
-          # https://opentelemetry.io/docs/collector/installation/#manual-linux-installation
-          "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v#{collector_version}/otelcol-contrib_#{collector_version}_#{host_os}_#{architecture}.tar.gz"
-        end
-
-        def destination
-          "#{context.config.value('collector_download_dir')}/otelcol.tar.gz"
-        end
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor/collector/extractor.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    module Collector
-      # Extracts the contents of the collector tar file.
-      class Extractor
-        attr_reader :context
-
-        def initialize(context)
-          @context = context
-        end
-
-        def extract!
-          # Already extracted. Noop.
-          return if has_been_extracted?
-
-          system("tar -xzf #{tar_path} -C #{context.config.value('collector_download_dir')}")
-        end
-
-        def has_been_extracted?
-          File.exist?(binary_path)
-        end
-
-        private
-
-        def tar_path
-          "#{context.config.value('collector_download_dir')}/otelcol.tar.gz"
-        end
-
-        def binary_path
-          "#{context.config.value('collector_download_dir')}/otelcol-contrib"
-        end
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor/collector/manager.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-require_relative './checksum'
-require_relative './configuration'
-require_relative './downloader'
-require_relative './extractor'
-
-module ScoutApm
-  module Logging
-    module Collector
-      # Manager class for the downloading, configuring, and starting of the collector.
-      class Manager
-        attr_reader :context
-
-        def initialize(context)
-          @context = context
-
-          @checksum = Checksum.new(@context)
-          @configuration = Configuration.new(@context)
-          @downloader = Downloader.new(@context)
-          @extractor = Extractor.new(@context)
-        end
-
-        def setup!
-          @configuration.setup!
-          @downloader.download!
-          @extractor.extract!
-
-          start_collector if verified_checksum_and_extracted?
-        end
-
-        def start_collector
-          context.logger.info('Starting otelcol-contrib')
-          collector_process = Process.spawn("#{extracted_collector_path}/otelcol-contrib --config #{config_file}")
-          File.write(context.config.value('collector_pid_file'), collector_process)
-        end
-
-        private
-
-        def verified_checksum_and_extracted?
-          has_verfied_checksum = @checksum.verified_checksum?(should_log_failures: true)
-          has_extracted_content = @extractor.has_been_extracted?
-
-          has_verfied_checksum && has_extracted_content
-        end
-
-        def extracted_collector_path
-          context.config.value('collector_download_dir')
-        end
-
-        def config_file
-          context.config.value('collector_config_file')
-        end
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor/monitor.rb

@@ -1,216 +0,0 @@
-# frozen_string_literal: true
-
-##
-# Launched as a daemon process by the monitor manager at Rails startup.
-##
-require 'json'
-
-require 'scout_apm'
-
-require_relative '../logger'
-require_relative '../context'
-require_relative '../config'
-require_relative '../utils'
-require_relative '../state'
-require_relative './collector/manager'
-
-module ScoutApm
-  module Logging
-    # Entry point for the monitor daemon process.
-    class Monitor
-      attr_reader :context
-      attr_accessor :latest_state_sha
-
-      @@instance = nil
-
-      def self.instance
-        @@instance ||= new
-      end
-
-      def initialize
-        @context = Context.new
-        context.logger.debug('Monitor instance created')
-
-        context.application_root = $stdin.gets&.chomp
-
-        # Load in the dynamic and state based config settings.
-        context.config = Config.with_file(context, determine_scout_config_filepath)
-
-        daemonize_process!
-      end
-
-      def setup!
-        context.config.logger.info('Monitor daemon process started')
-
-        add_exit_handler!
-
-        unless has_logs_to_monitor?
-          context.config.logger.warn('No logs are set to be monitored. Please set the `logs_monitored` config setting. Exiting.')
-          return
-        end
-
-        initiate_collector_setup! unless has_previous_collector_setup?
-
-        @latest_state_sha = get_state_file_sha
-
-        run!
-      end
-
-      def run!
-        # Prevent the monitor from checking the collector health before it's fully started.
-        # Having this be configurable is useful for testing.
-        sleep context.config.value('monitor_interval_delay')
-
-        loop do
-          sleep context.config.value('monitor_interval')
-
-          check_collector_health
-
-          check_state_change
-        end
-      end
-
-      # Only useful for testing.
-      def config=(config)
-        context.config = config
-      end
-
-      private
-
-      def daemonize_process!
-        # Similar to that of Process.daemon, but we want to keep the dir, STDOUT and STDERR.
-        exit if fork
-        Process.setsid
-        exit if fork
-        $stdin.reopen '/dev/null'
-
-        context.logger.debug("Monitor process daemonized, PID: #{Process.pid}")
-        File.write(context.config.value('monitor_pid_file'), Process.pid)
-      end
-
-      def has_logs_to_monitor?
-        context.config.value('logs_monitored').any?
-      end
-
-      def has_previous_collector_setup?
-        return false unless context.config.value('health_check_port') != 0
-
-        healthy_response = request_health_check_port("http://localhost:#{context.config.value('health_check_port')}/")
-
-        if healthy_response
-          context.logger.info("Collector already setup on port #{context.config.value('health_check_port')}")
-        else
-          context.logger.info('Setting up new collector')
-        end
-
-        healthy_response
-      end
-
-      def initiate_collector_setup!
-        set_health_check_port!
-
-        Collector::Manager.new(context).setup!
-      end
-
-      def is_port_available?(port)
-        socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
-        remote_address = Socket.sockaddr_in(port, '127.0.0.1')
-
-        begin
-          socket.connect_nonblock(remote_address)
-        rescue Errno::EINPROGRESS
-          IO.select(nil, [socket])
-          retry
-        rescue Errno::EISCONN, Errno::ECONNRESET
-          false
-        rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
-          true
-        ensure
-          socket.close if socket && !socket.closed?
-        end
-      end
-
-      def set_health_check_port!
-        health_check_port = 13_133
-        until is_port_available?(health_check_port)
-          sleep 0.1
-          health_check_port += 1
-        end
-
-        Config::ConfigDynamic.set_value('health_check_port', health_check_port)
-        context.config.state.flush_state!
-      end
-
-      def request_health_check_port(endpoint)
-        uri = URI(endpoint)
-
-        begin
-          response = Net::HTTP.get_response(uri)
-
-          unless response.is_a?(Net::HTTPSuccess)
-            context.logger.error("Error occurred while checking collector health: #{response.message}")
-            return false
-          end
-        rescue StandardError => e
-          context.logger.error("Error occurred while checking collector health: #{e.message}")
-          return false
-        end
-
-        true
-      end
-
-      def check_collector_health
-        context.logger.debug('Checking collector health')
-        collector_health_endpoint = "http://localhost:#{context.config.value('health_check_port')}/"
-
-        healthy_response = request_health_check_port(collector_health_endpoint)
-
-        initiate_collector_setup! unless healthy_response
-      end
-
-      def remove_collector_process # rubocop:disable Metrics/AbcSize
-        return unless File.exist? context.config.value('collector_pid_file')
-
-        process_id = File.read(context.config.value('collector_pid_file'))
-        return if process_id.empty?
-
-        begin
-          Process.kill('TERM', process_id.to_i)
-        rescue Errno::ENOENT, Errno::ESRCH => e
-          context.logger.error("Error occurred while removing collector process from monitor: #{e.message}")
-        ensure
-          File.delete(context.config.value('collector_pid_file'))
-        end
-      end
-
-      def check_state_change
-        current_sha = get_state_file_sha
-
-        return if current_sha == latest_state_sha
-
-        remove_collector_process
-        initiate_collector_setup!
-
-        # File SHA can change due to port mappings on collector setup.
-        @latest_state_sha = get_state_file_sha
-      end
-
-      def add_exit_handler!
-        at_exit do
-          # There may not be a file to delete, as the monitor manager ensures cleaning it up when monitoring is disabled.
-          File.delete(context.config.value('monitor_pid_file')) if File.exist?(context.config.value('monitor_pid_file'))
-        end
-      end
-
-      def get_state_file_sha
-        return nil unless File.exist?(context.config.value('monitor_state_file'))
-
-        `sha256sum #{context.config.value('monitor_state_file')}`.split(' ').first
-      end
-
-      def determine_scout_config_filepath
-        "#{context.application_root}/config/scout_apm.yml"
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor_manager/manager.rb

@@ -1,162 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    # Manages the creation of the daemon monitor process.
-    class MonitorManager
-      attr_reader :context
-
-      @@instance = nil
-
-      def self.instance
-        @@instance ||= new
-      end
-
-      def initialize
-        @context = Context.new
-        context.config = Config.with_file(context, context.config.value('config_file'))
-      end
-
-      def setup!
-        context.config.log_settings(context.logger)
-        context.logger.info('Setting up monitor daemon process')
-
-        add_exit_handler!
-
-        determine_configuration_state
-      end
-
-      def determine_configuration_state
-        monitoring_enabled = context.config.value('logs_monitor')
-
-        if monitoring_enabled
-          context.logger.info('Log monitoring enabled')
-          create_process
-
-          # Continue to hold the lock until we have written the PID file.
-          ensure_monitor_pid_file_exists
-        else
-          context.logger.info('Log monitoring disabled')
-          remove_processes
-        end
-      end
-
-      # With the use of fileoffsets in the collector, and the persistent queue of already collected logs,
-      # we can safely restart the collector. Due to the way fingerprinting of the files works, if the
-      # file path switches, but the beginning contents of the file remain the same, the file will be
-      # treated as the same file as before.
-      # If logs get rotated, the fingerprint changes, and the collector automatically detects this.
-      def add_exit_handler!
-        # With the use of unicorn and puma worker killer, we want to ensure we only restart (exit and
-        # eventually start) the monitor and collector when the main process exits, and not the workers.
-        initialized_process_id = Process.pid
-        at_exit do
-          # Only remove/restart the monitor and collector if we are exiting from an app_server process.
-          # We need to wait on this check, as the process command line changes at some point.
-          if Utils.current_process_is_app_server? && Process.pid == initialized_process_id
-            context.logger.debug('Exiting from app server process. Removing monitor and collector processes.')
-            remove_processes
-          end
-        end
-      end
-
-      def create_process
-        return if process_exists?
-
-        Utils.ensure_directory_exists(context.config.value('monitor_pid_file'))
-
-        reader, writer = IO.pipe
-
-        gem_directory = File.expand_path('../../../..', __dir__)
-
-        # As we daemonize the process, we will write to the pid file within the process.
-        pid = Process.spawn("ruby #{gem_directory}/bin/scout_apm_logging_monitor", in: reader)
-
-        reader.close
-        # TODO: Add support for Sinatra.
-        writer.puts Rails.root if defined?(Rails)
-        writer.close
-        # Block until we have spawned the process and forked. This is to ensure
-        # we keep the exclusive lock until the process has written the PID file.
-        Process.wait(pid)
-      end
-
-      private
-
-      def ensure_monitor_pid_file_exists
-        start_time = Time.now
-        # We don't want to hold up the initial Rails boot time for very long.
-        timeout_seconds = 0.1
-
-        # Naive benchmarks show this taking ~0.01 seconds.
-        loop do
-          if File.exist?(context.config.value('monitor_pid_file'))
-            context.logger.debug('Monitor PID file exists. Releasing lock.')
-            break
-          end
-
-          if Time.now - start_time > timeout_seconds
-            context.logger.warn('Unable to verify monitor PID file write. Releasing lock.')
-            break
-          end
-
-          sleep 0.01
-        end
-      end
-
-      def process_exists?
-        return false unless File.exist? context.config.value('monitor_pid_file')
-
-        process_id = File.read(context.config.value('monitor_pid_file'))
-        return false if process_id.empty?
-
-        process_exists = Utils.check_process_liveliness(process_id.to_i, 'scout_apm_logging_monitor')
-        File.delete(context.config.value('monitor_pid_file')) unless process_exists
-
-        process_exists
-      end
-
-      def remove_monitor_process # rubocop:disable Metrics/AbcSize
-        return unless File.exist? context.config.value('monitor_pid_file')
-
-        process_id = File.read(context.config.value('monitor_pid_file'))
-        return if process_id.empty?
-
-        begin
-          Process.kill('TERM', process_id.to_i)
-        rescue Errno::ENOENT, Errno::ESRCH => e
-          context.logger.error("Error occurred while removing monitor process: #{e.message}")
-          File.delete(context.config.value('monitor_pid_file'))
-        end
-      end
-
-      def remove_collector_process # rubocop:disable Metrics/AbcSize
-        return unless File.exist? context.config.value('collector_pid_file')
-
-        process_id = File.read(context.config.value('collector_pid_file'))
-        return if process_id.empty?
-
-        begin
-          Process.kill('TERM', process_id.to_i)
-        rescue Errno::ENOENT, Errno::ESRCH => e
-          context.logger.error("Error occurred while removing collector process from manager: #{e.message}")
-        ensure
-          File.delete(context.config.value('collector_pid_file'))
-        end
-      end
-
-      def remove_data_file
-        return unless File.exist? context.config.value('monitor_state_file')
-
-        File.delete(context.config.value('monitor_state_file'))
-      end
-
-      # Remove both the monitor and collector processes that we have spawned.
-      def remove_processes
-        remove_monitor_process
-        remove_collector_process
-        remove_data_file
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/state.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    class Config
-      # Responsibling for ensuring safe interprocess persitance around configuration state.
-      class State
-        attr_reader :context
-
-        def initialize(context)
-          @context = context
-        end
-
-        def load_state_from_file
-          return unless File.exist?(context.config.value('monitor_state_file'))
-
-          file_contents = File.read(context.config.value('monitor_state_file'))
-          JSON.parse(file_contents)
-        end
-
-        def flush_to_file!(updated_log_locations = []) # rubocop:disable Metrics/AbcSize
-          Utils.ensure_directory_exists(context.config.value('monitor_state_file'))
-
-          File.open(context.config.value('monitor_state_file'), (File::RDWR | File::CREAT), 0o644) do |file|
-            file.flock(File::LOCK_EX)
-
-            data = Config::ConfigState.get_values_to_set.each_with_object({}) do |key, memo|
-              memo[key] = context.config.value(key)
-            end
-
-            contents = file.read
-            old_log_state_files = if contents.empty?
-                                    []
-                                  else
-                                    current_data = JSON.parse(contents)
-                                    current_data['logs_monitored']
-                                  end
-
-            data['logs_monitored'] =
-              merge_and_dedup_log_locations(updated_log_locations, old_log_state_files, data['logs_monitored'])
-
-            file.rewind # Move cursor to beginning of the file
-            file.truncate(0) # Truncate existing content
-            file.write(JSON.pretty_generate(data))
-          rescue StandardError => e
-            context.logger.error("Error occurred while flushing state to file: #{e.message}. Unlocking.")
-          ensure
-            file.flock(File::LOCK_UN)
-          end
-        end
-
-        private
-
-        # Should we add better detection for similar basenames but different paths?
-        # May be a bit tricky with tools like capistrano and releases paths differentiated by time.
-        def merge_and_dedup_log_locations(*log_locations)
-          # Take the new logs if duplication (those first passed in the args) as we could be in a newer release.
-          logs = log_locations.reduce([], :concat)
-          merged = logs.each_with_object({}) do |log_path, hash|
-            base_name = File.basename(log_path)
-            hash[base_name] ||= log_path
-          end
-
-          merged.values
-        end
-      end
-    end
-  end
-end

data/spec/data/logs_config.yml

@@ -1,3 +0,0 @@
-extensions:
-  file_storage/otc:
-    directory: /dev/null

data/spec/data/state_file.json

@@ -1,3 +0,0 @@
-{
-  "health_check_port": 1234
-}