scout_apm_logging 0.0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 198bf309d17f0abfcf2c33f820a2d44e1e92ef5af9e321ee9fbaacfec256f032
+  data.tar.gz: 2a09b14a45e702fe1e4f4caee6ff89475e14971ba9edad41ae5cf3c72ab97850
+SHA512:
+  metadata.gz: ebd30ceac50e2f14f3693946e8dd883112cc502513678b3a9963e253f2622d1ea488c2a66bce01f3e20726d7c6f08f61392cde6f22f1d9d0412987780f6399fa
+  data.tar.gz: c0cfeb3cfbed2628772550b18b1a2f7965a144fafea14846ed65936043ae283b48cef6ba0109a05dfe9df3b826080d0f24676c0957b3909311cae12098b1e2de

data/.github/workflows/test.yml

@@ -0,0 +1,37 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    name: RSpec
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ['2.6', '2.7', '3.0', '3.1', '3.2', '3.3' ]
+    env:
+      DOCKER_RUBY_VERSION: ${{ matrix.ruby }}
+      BUNDLE_GEMFILE: gems/rails.gemfile
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run RSpec
+        run: bundle exec rake test
+
+  rubocop:
+    name: RuboCop
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.2"
+          bundler-cache: true
+      - name: Run check
+        run: bundle exec rubocop

data/.gitignore

@@ -0,0 +1,13 @@
+# Ignore built gems.
+*.gem
+# Ignore lock files.
+Gemfile.lock
+gems/*.lock
+# Ignore .idea
+.idea
+# Ignore log files
+log/*.log
+# Ignore checksum files
+tooling/*
+!tooling/*.md
+!tooling/*.rb

data/.rubocop.yml

@@ -0,0 +1,65 @@
+AllCops:
+  Exclude:
+    - "bin/*"
+    - "tmp/**/*"
+    - "vendor/**/*"
+    - "spec/spec_helper.rb"
+    - "gems/*"
+    - "Rakefile"
+    - "tooling/*"
+  NewCops: disable
+  TargetRubyVersion: 2.6
+
+Metrics/BlockLength:
+  Exclude:
+    - "spec/**/*"
+
+Metrics/ClassLength:
+  Max: 150
+
+Style/FrozenStringLiteralComment:
+  EnforcedStyle: always
+  Exclude:
+    - "Gemfile*"
+    - "*.gemspec"
+    - "spec/**/*"
+
+Naming/FileName:
+  Exclude:
+    - "lib/scout_apm/scout_apm_logging.rb"
+    - "spec/scout_apm_logging_spec.rb"
+
+Style/Documentation:
+  Enabled: true
+
+Naming/PredicateName:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Style/ClassVars:
+  Enabled: false
+
+Style/EmptyElse:
+  Enabled: false
+
+Naming/AccessorMethodName:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 25
+
+Lint/LiteralAsCondition:
+  Enabled: false
+
+Layout/LineLength:
+  Max: 130
+
+Style/PercentLiteralDelimiters:
+  PreferredDelimiters:
+    '%w': "[]"
+    '%W': "[]"

data/Dockerfile

@@ -0,0 +1,18 @@
+ARG RUBY_VERSION=3.3
+FROM ruby:$RUBY_VERSION
+
+# Set the working directory inside the container
+WORKDIR /app
+
+ARG BUNDLE_GEMFILE=./gems/rails.gemfile
+
+# # Copy the entire project directory into the container
+COPY . .
+
+# Remove any local .lock files after copying
+RUN find . -type f -name "*.lock" -exec rm -f {} \;
+
+# Install dependencies
+RUN bundle install
+
+ENV BUNDLE_GEMFILE=./gems/rails.gemfile

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'rake', '~> 13.0'

data/README.md

@@ -0,0 +1,58 @@
+# ScoutApm Ruby Logging
+
+[![Build Status](https://github.com/scoutapp/scout_apm_ruby_logging/actions/workflows/test.yml/badge.svg)](https://github.com/scoutapp/scout_apm_ruby_logging/actions)
+
+A Ruby gem for detailed, easy to navigate, managed log monitoring.
+
+Sign up for an account at https://www.scoutapm.com to start monitoring your logs and application performance in minutes.
+
+## Getting Started
+Add the gem to your Gemfile: 
+
+```ruby
+gem 'scout_apm_logging'
+```
+
+Update your Gemfile: 
+```ruby
+bundle install
+```
+
+Update your [RAILS_ROOT/config/scout_apm.yml](https://scoutapm.com/apps/new_ruby_application_configuration) and add the following:
+
+```yaml
+  # ... Previous &defaults or environment defined configurations
+
+  # ENV equivalent: SCOUT_LOGS_MONITOR=true
+  # ENV equivalent: SCOUT_LOGS_INGEST_KEY=...
+
+  logs_monitor: true
+  logs_ingest_key: ...
+```
+
+Deploy :rocket:
+
+## Testing
+To run the entire test suite:
+```ruby
+bundle exec rake test
+```
+
+To run an individual test file within the suite:
+```ruby
+bundle exec rake test file=/path/to/spec/_spec.rb
+```
+
+To run test(s) against a specific Ruby version:
+```ruby
+DOCKER_RUBY_VERSION=3.3 bundle exec rake test
+```
+
+## Local
+Point your Gemfile at your local checkout: 
+```ruby
+gem 'scout_apm_logging', path: '/path/to/scout_apm_ruby_logging'
+```
+
+## Help
+Email support@scoutapm.com if you need a hand.

data/Rakefile

@@ -0,0 +1,35 @@
+task :default => :test
+
+task :test do
+  ruby_version = ENV.has_key?("DOCKER_RUBY_VERSION") ? ENV["DOCKER_RUBY_VERSION"] : "3.3"
+
+  unless system("docker image inspect rspec-runner-#{ruby_version} > /dev/null 2>&1")
+    puts "Building RSpec runner Docker image..."
+    system("docker build --build-arg RUBY_VERSION=#{ruby_version} -t rspec-runner-#{ruby_version} .")
+  end
+
+  additional_options = ENV["debug"] ? "-it" : ""
+
+  if ENV["file"]
+    puts "Running RSpec test for #{ENV["file"]}..."
+    system("docker run -v #{Dir.pwd}/lib:/app/lib -v #{Dir.pwd}/spec:/app/spec #{additional_options} --rm rspec-runner-#{ruby_version} bundle exec rspec #{ENV["file"]} 2>&1")
+  else
+    puts "Running RSpec tests..."
+    Dir.glob("spec/**/*_spec.rb") do |spec_file|
+      puts "Running #{spec_file}..."
+      system("docker run -v #{Dir.pwd}/lib:/app/lib -v #{Dir.pwd}/spec:/app/spec #{additional_options} --rm rspec-runner-#{ruby_version} bundle exec rspec #{spec_file} 2>&1")
+
+      # Exit the task if a test fails
+      unless $?.success?
+        abort("Test failed: #{spec_file}")
+      end
+    end
+  end
+end
+
+task :access_container do
+  puts "Accessing a new rspec-runner-#{ruby_version} container..."
+
+  ruby_version = ENV.has_key?("DOCKER_RUBY_VERSION") ? ENV["DOCKER_RUBY_VERSION"] : "3.3"
+  system("docker run -it --rm -v #{Dir.pwd}/lib:/app/lib -v #{Dir.pwd}/spec:/app/spec --entrypoint /bin/bash rspec-runner-#{ruby_version}")
+end

data/bin/scout_apm_logging_monitor

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/scout_apm/logging/monitor/monitor.rb"
+
+ScoutApm::Logging::Monitor.instance.setup!

data/gems/rails.gemfile

@@ -0,0 +1,3 @@
+eval_gemfile("../Gemfile")
+
+gem "rails"

data/lib/scout_apm/logging/config.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+# Valid Config Options:
+#
+# This list is complete, but some are for developers of
+# scout_apm_logging itself. See the documentation at https://scoutapm.com/docs for
+# customer-focused documentation.
+#
+# config_file - location of the scout_apm.yml configuration file
+# log_level          - log level for the internal library itself
+# log_stdout         - true or false.  If true, log to STDOUT
+# log_stderr         - true or false.  If true, log to STDERR
+# log_file_path      - either a directory or "STDOUT"
+# log_class          - the underlying class to use for logging.  Defaults to Ruby's Logger class
+# logs_monitor       - true or false.  If true, monitor logs
+# logs_monitored     - an array of log file paths to monitor. Overrides the default log destination detection
+# logs_ingest_key    - the ingest key to use for logs
+# logs_capture_level - the minimum log level to start capturing logs for
+# logs_config        - a hash of configuration options for merging into the collector's config
+# logs_reporting_endpoint - the endpoint to send logs to
+# logs_proxy_log_dir - the directory to store logs in for monitoring
+# manager_lock_file  - the location for obtaining an exclusive lock for running monitor manager
+# monitor_pid_file   - the location of the pid file for the monitor
+# monitor_state_file - the location of the state file for the monitor
+# monitor_interval   - the interval to check the collector healtcheck and for new state logs
+# monitor_interval_delay - the delay to wait before running the first monitor interval
+# collector_log_level - the log level for the collector
+# collector_sending_queue_storage_dir - the directory to store queue files
+# collector_offset_storage_dir - the directory to store offset files
+# collector_pid_file - the location of the pid file for the collector
+# collector_download_dir - the directory to store downloaded collector files
+# collector_config_file - the location of the config file for the collector
+# collector_version - the version of the collector to download
+# health_check_port - the port to use for the collector health check. Default is dynamically derived based on port availability
+#
+# Any of these config settings can be set with an environment variable prefixed
+# by SCOUT_ and uppercasing the key: SCOUT_LOG_LEVEL for instance.
+
+module ScoutApm
+  module Logging
+    # Holds the configuration values for Scout APM Logging.
+    class Config < ScoutApm::Config
+      KNOWN_CONFIG_OPTIONS = %w[
+        config_file
+        log_level
+        log_stderr
+        log_stdout
+        log_file_path
+        log_class
+        logs_monitor
+        logs_monitored
+        logs_ingest_key
+        logs_capture_level
+        logs_config
+        logs_reporting_endpoint
+        logs_proxy_log_dir
+        manager_lock_file
+        monitor_pid_file
+        monitor_state_file
+        monitor_interval
+        monitor_interval_delay
+        collector_sending_queue_storage_dir
+        collector_offset_storage_dir
+        collector_pid_file
+        collector_download_dir
+        collector_config_file
+        collector_log_level
+        collector_version
+        health_check_port
+      ].freeze
+
+      SETTING_COERCIONS = {
+        'logs_monitor' => BooleanCoercion.new,
+        'logs_monitored' => JsonCoercion.new,
+        'monitor_interval' => IntegerCoercion.new,
+        'monitor_interval_delay' => IntegerCoercion.new,
+        'health_check_port' => IntegerCoercion.new
+      }.freeze
+
+      # The bootstrapped, and initial config that we attach to the context. Will be swapped out by
+      # both the monitor and manager on initialization to the one with a file (which also has the dynamic
+      # and state configs).
+      def self.without_file(context)
+        overlays = [
+          ConfigEnvironment.new,
+          ConfigDefaults.new,
+          ConfigNull.new
+        ]
+        new(context, overlays)
+      end
+
+      def self.with_file(context, file_path = nil, config = {})
+        overlays = [
+          ConfigEnvironment.new,
+          ConfigFile.new(context, file_path, config),
+          ConfigDynamic.new,
+          ConfigState.new(context),
+          ConfigDefaults.new,
+          ConfigNull.new
+        ]
+        new(context, overlays)
+      end
+
+      def state
+        @overlays.find { |overlay| overlay.is_a? ConfigState }
+      end
+
+      def value(key)
+        unless KNOWN_CONFIG_OPTIONS.include?(key)
+          logger.debug("Requested looking up a unknown configuration key: #{key} (not a problem. Evaluate and add to config.rb)")
+        end
+
+        o = overlay_for_key(key)
+        raw_value = if o
+                      o.value(key)
+                    else
+                      # No overlay said it could handle this key, bail out with nil.
+                      nil
+                    end
+
+        coercion = SETTING_COERCIONS.fetch(key, NullCoercion.new)
+        coercion.coerce(raw_value)
+      end
+
+      def all_settings
+        KNOWN_CONFIG_OPTIONS.inject([]) do |memo, key|
+          o = overlay_for_key(key)
+          memo << { key: key, value: value(key).inspect, source: o.name }
+        end
+      end
+
+      # Dynamically set state based on the application.
+      class ConfigDynamic
+        @values_to_set = {
+          'health_check_port': nil
+        }
+
+        class << self
+          attr_reader :values_to_set
+
+          def set_value(key, value)
+            @values_to_set[key] = value
+          end
+        end
+
+        def value(key)
+          self.class.values_to_set[key]
+        end
+
+        def has_key?(key)
+          self.class.values_to_set.key?(key)
+        end
+
+        def name
+          'dynamic'
+        end
+      end
+
+      # State that is persisted and communicated upon by multiple processes.
+      class ConfigState
+        @values_to_set = {
+          'logs_monitored': [],
+          'health_check_port': nil
+        }
+
+        class << self
+          attr_reader :values_to_set
+
+          def set_value(key, value)
+            @values_to_set[key] = value
+          end
+
+          def get_values_to_set
+            @values_to_set.keys.map(&:to_s)
+          end
+        end
+
+        attr_reader :context, :state
+
+        def initialize(context)
+          @context = context
+
+          # Note, the config on the context we are passing in here comes from the Config.without_file. We
+          # won't be aware of a state file that was defined in a config file, but this would be a very
+          # rare thing to have happen as this is more of an internal config value.
+          @state = State.new(context)
+
+          set_values_from_state
+        end
+
+        def value(key)
+          self.class.values_to_set[key]
+        end
+
+        def has_key?(key)
+          self.class.values_to_set.key?(key)
+        end
+
+        def name
+          'state'
+        end
+
+        def flush_state!
+          state.flush_to_file!
+        end
+
+        def add_log_locations!(updated_log_locations)
+          state.flush_to_file!(updated_log_locations)
+        end
+
+        private
+
+        def set_values_from_state
+          data = state.load_state_from_file
+
+          return unless data
+
+          data.each do |key, value|
+            self.class.set_value(key, value)
+          end
+        end
+      end
+
+      # Defaults in case no config file has been found.
+      class ConfigDefaults
+        DEFAULTS = {
+          'log_level' => 'info',
+          'logs_monitored' => [],
+          'logs_capture_level' => 'debug',
+          'logs_reporting_endpoint' => 'https://otlp.telemetryhub.com:4317',
+          'logs_proxy_log_dir' => '/tmp/scout_apm/logs/',
+          'manager_lock_file' => '/tmp/scout_apm/monitor_lock_file.lock',
+          'monitor_pid_file' => '/tmp/scout_apm/scout_apm_log_monitor.pid',
+          'monitor_state_file' => '/tmp/scout_apm/scout_apm_log_monitor_state.json',
+          'monitor_interval' => 60,
+          'monitor_interval_delay' => 60,
+          'collector_log_level' => 'error',
+          'collector_offset_storage_dir' => '/tmp/scout_apm/file_storage/receiver/',
+          'collector_sending_queue_storage_dir' => '/tmp/scout_apm/file_storage/otc/',
+          'collector_pid_file' => '/tmp/scout_apm/scout_apm_otel_collector.pid',
+          'collector_download_dir' => '/tmp/scout_apm/',
+          'collector_config_file' => '/tmp/scout_apm/config.yml',
+          'collector_version' => '0.102.1'
+        }.freeze
+
+        def value(key)
+          DEFAULTS[key]
+        end
+
+        def has_key?(key)
+          DEFAULTS.key?(key)
+        end
+
+        # Dyanmic/computed values are here, but not counted as user specified.
+        def any_keys_found?
+          false
+        end
+
+        def name
+          'defaults'
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/context.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    # Contains context around Scout APM logging, such as environment, configuration, and the logger.
+    class Context
+      # The root of the application.
+      attr_accessor :application_root
+
+      # Initially start up without attempting to load a configuration file. We
+      # need to be able to lookup configuration options like "application_root"
+      # which would then in turn influence where the yaml configuration file is
+      # located
+      #
+      # Later in initialization, we set config= to include the file.
+      def initialize
+        @logger = LoggerFactory.build_minimal_logger
+      end
+
+      def config
+        @config ||= Config.without_file(self)
+      end
+
+      def environment
+        @environment ||= ScoutApm::Environment.instance
+      end
+
+      def logger
+        @logger ||= LoggerFactory.build(config, environment, application_root)
+      end
+
+      def config=(config)
+        @config = config
+
+        @logger = nil
+      end
+    end
+
+    # Create a logger based on the configuration settings.
+    class LoggerFactory
+      def self.build(config, environment, application_root = nil)
+        root = application_root || environment.root
+        Logger.new(root,
+                   {
+                     log_level: config.value('log_level'),
+                     log_file_path: config.value('log_file_path'),
+                     stdout: config.value('log_stdout') || environment.platform_integration.log_to_stdout?,
+                     stderr: config.value('log_stderr'),
+                     logger_class: config.value('log_class')
+                   })
+      end
+
+      def self.build_minimal_logger
+        Logger.new(nil, stdout: true)
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/logger.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    # Custom internal logger for ScoutApm Logging.
+    class Logger < ScoutApm::Logger
+      private
+
+      def determine_log_destination
+        case true
+        when stdout?
+          $stdout
+        when stderr?
+          $stderr
+        when validate_path(@opts[:log_file])
+          @opts[:log_file]
+        when validate_path("#{log_file_path}/scout_apm_logging.log")
+          "#{log_file_path}/scout_apm_logging.log"
+        else
+          # Safe fallback
+          $stdout
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/loggers/capture.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+require_relative './swap'
+require_relative './proxy'
+
+module ScoutApm
+  module Logging
+    module Loggers
+      # Will capture the log destinations from the application's loggers.
+      class Capture
+        attr_reader :context
+
+        def initialize(context)
+          @context = context
+        end
+
+        def capture_log_locations! # rubocop:disable Metrics/AbcSize
+          logger_instances << Rails.logger if defined?(Rails)
+          logger_instances << Sidekiq.logger if defined?(Sidekiq)
+          logger_instances << ObjectSpace.each_object(::ScoutTestLogger).to_a if defined?(::ScoutTestLogger)
+
+          # Swap in our logger for each logger instance, in conjunction with the original class.
+          updated_log_locations = logger_instances.compact.flatten.map do |logger|
+            swapped_in_location(logger)
+          end
+
+          context.config.state.add_log_locations!(updated_log_locations)
+        end
+
+        private
+
+        def logger_instances
+          @logger_instances ||= []
+        end
+
+        def swapped_in_location(log_instance)
+          swap = Swap.new(context, log_instance)
+          swap.update_logger!
+          swap.log_location
+        end
+      end
+    end
+  end
+end