scout_apm_logging 0.0.12 → 1.0.0

data/spec/integration/loggers/capture_spec.rb

@@ -1,68 +0,0 @@
-require 'logger'
-
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/loggers/capture'
-
-describe ScoutApm::Logging::Loggers::Capture do
-  it 'should find the logger, capture the log destination, and rotate collector configs' do
-    ENV['SCOUT_MONITOR_INTERVAL'] = '10'
-    ENV['SCOUT_MONITOR_INTERVAL_DELAY'] = '10'
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-
-    context = ScoutApm::Logging::MonitorManager.instance.context
-
-    state_file_location = context.config.value('monitor_state_file')
-    collector_pid_location = context.config.value('collector_pid_file')
-    ScoutApm::Logging::Utils.ensure_directory_exists(state_file_location)
-
-    first_logger = ScoutTestLogger.new('/tmp/first_file.log')
-    first_logger_basename = File.basename(first_logger.instance_variable_get(:@logdev).filename.to_s)
-    first_logger_updated_path = File.join(context.config.value('logs_proxy_log_dir'), first_logger_basename)
-    TestLoggerWrapper.logger = first_logger
-
-    similuate_railtie
-
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-    collector_pid = File.read(collector_pid_location)
-
-    content = File.read(state_file_location)
-    data = JSON.parse(content)
-    expect(data['logs_monitored']).to eq([first_logger_updated_path])
-
-    second_logger = ScoutTestLogger.new('/tmp/second_file.log')
-    second_logger_basename = File.basename(second_logger.instance_variable_get(:@logdev).filename.to_s)
-    second_logger_updated_path = File.join(context.config.value('logs_proxy_log_dir'), second_logger_basename)
-    TestLoggerWrapper.logger = second_logger
-
-    similuate_railtie
-
-    content = File.read(state_file_location)
-    data = JSON.parse(content)
-
-    expect(data['logs_monitored'].sort).to eq([first_logger_updated_path, second_logger_updated_path])
-
-    # Need to wait for the delay first health check, next monitor interval to restart the collector, and then for
-    # the collector to restart
-    sleep 25
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-    new_collector_pid = File.read(collector_pid_location)
-
-    # Should have restarted the collector based on the change
-    expect(new_collector_pid).not_to eq(collector_pid)
-  end
-
-  private
-
-  def similuate_railtie
-    context = ScoutApm::Logging::MonitorManager.instance.context
-
-    ScoutApm::Logging::Loggers::Capture.new(context).setup!
-    ScoutApm::Logging::MonitorManager.new.setup!
-  end
-end

data/spec/integration/monitor/collector/downloader/will_verify_checksum.rb

@@ -1,49 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../../../lib/scout_apm/logging/monitor/collector/downloader'
-
-describe ScoutApm::Logging::Collector::Downloader do
-  it 'should validate checksum, and correct download if neccessary' do
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    otelcol_contrib_path = '/tmp/scout_apm/otelcol-contrib'
-    ScoutApm::Logging::Utils.ensure_directory_exists(otelcol_contrib_path)
-
-    File.write(otelcol_contrib_path, 'fake content')
-
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    download_time = File.mtime(otelcol_contrib_path)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-
-    ENV['SCOUT_LOGS_MONITOR'] = 'false'
-
-    ScoutApm::Logging::MonitorManager.new.setup!
-
-    sleep 5 # Give the process time to exit
-
-    expect(File.exist?(ScoutApm::Logging::MonitorManager.instance.context.config.value('monitor_pid_file'))).to be_falsey
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).to be_empty
-    expect(`pgrep scout_apm_log_monitor --runstates D,R,S`).to be_empty
-
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-
-    ScoutApm::Logging::MonitorManager.new.setup!
-
-    # Give the process time to exit, and for the healthcheck to restart it
-    wait_for_process_with_timeout!('otelcol-contrib', 30)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-
-    recheck_time = File.mtime(otelcol_contrib_path)
-
-    expect(download_time).to eq(recheck_time)
-  end
-end

data/spec/integration/monitor/collector_healthcheck_spec.rb

@@ -1,29 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/monitor/monitor'
-
-describe ScoutApm::Logging::Monitor do
-  it 'should recreate collector process on healthcheck if it has exited' do
-    ENV['SCOUT_MONITOR_INTERVAL'] = '10'
-    ENV['SCOUT_MONITOR_INTERVAL_DELAY'] = '10'
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    ScoutApm::Logging::Utils.ensure_directory_exists('/tmp/scout_apm/scout_apm_log_monitor.pid')
-
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-
-    # Bypass gracefull shutdown
-    `pkill -9 otelcol-contrib`
-
-    # Give the process time to exit, and for the healthcheck to restart it
-    wait_for_process_with_timeout!('otelcol-contrib', 30)
-  end
-end

data/spec/integration/monitor/continuous_state_collector_spec.rb

@@ -1,31 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/monitor/monitor'
-
-describe ScoutApm::Logging::Monitor do
-  it "Should not restart the collector if the state hasn't changed" do
-    ENV['SCOUT_MONITOR_INTERVAL'] = '10'
-    ENV['SCOUT_MONITOR_INTERVAL_DELAY'] = '10'
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    context = ScoutApm::Logging::MonitorManager.instance.context
-    collector_pid_location = context.config.value('collector_pid_file')
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-    collector_pid = File.read(collector_pid_location)
-
-    # Give time for the monitor interval to run.
-    sleep 30
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).not_to be_empty
-    second_read_pid = File.read(collector_pid_location)
-
-    expect(second_read_pid).to eq(collector_pid)
-  end
-end

data/spec/integration/monitor/previous_collector_setup_spec.rb

@@ -1,45 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/monitor/monitor'
-
-describe ScoutApm::Logging::Monitor do
-  it 'should use previous collector setup if monitor daemon exits' do
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    monitor_pid_location = ScoutApm::Logging::MonitorManager.instance.context.config.value('monitor_pid_file')
-    collector_pid_location = ScoutApm::Logging::MonitorManager.instance.context.config.value('collector_pid_file')
-    ScoutApm::Logging::Utils.ensure_directory_exists(monitor_pid_location)
-
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    monitor_pid = File.read(monitor_pid_location)
-
-    otelcol_pid = `pgrep otelcol-contrib --runstates D,R,S`.strip!
-    stored_otelcol_pid = File.read(collector_pid_location)
-    expect(otelcol_pid).to eq(stored_otelcol_pid)
-
-    `kill -9 #{monitor_pid}`
-
-    # Create a separate monitor manager instance, or else we won't reload
-    # the configuraiton state.
-    ScoutApm::Logging::MonitorManager.new.setup!
-
-    sleep 5
-
-    expect(`pgrep -f /app/bin/scout_apm_logging_monitor --runstates D,R,S`).not_to be_empty
-
-    new_monitor_pid = File.read(monitor_pid_location)
-    expect(new_monitor_pid).not_to eq(monitor_pid)
-
-    should_be_same_otelcol_pid = `pgrep otelcol-contrib --runstates D,R,S`.strip!
-    should_be_same_stored_otelcol_pid = File.read(collector_pid_location)
-    expect(should_be_same_otelcol_pid).to eq(otelcol_pid)
-    expect(should_be_same_stored_otelcol_pid).to eq(stored_otelcol_pid)
-  end
-end

data/spec/integration/monitor_manager/disable_agent_spec.rb

@@ -1,30 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/monitor/collector/manager'
-
-describe ScoutApm::Logging::Collector::Manager do
-  it 'If monitor is false, it should remove the daemon and collector process if they are present' do
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).to be_empty
-
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    ENV['SCOUT_LOGS_MONITOR'] = 'false'
-
-    ScoutApm::Logging::MonitorManager.new.setup!
-
-    sleep 5 # Give the process time to exit
-
-    expect(File.exist?(ScoutApm::Logging::MonitorManager.instance.context.config.value('monitor_pid_file'))).to be_falsey
-    expect(`pgrep otelcol-contrib --runstates D,R,S`).to be_empty
-    expect(`pgrep scout_apm_log_monitor --runstates D,R,S`).to be_empty
-
-    ENV.delete('SCOUT_LOGS_MONITOR')
-  end
-end

data/spec/integration/monitor_manager/monitor_pid_file_spec.rb

@@ -1,38 +0,0 @@
-require 'spec_helper'
-
-require_relative '../../../lib/scout_apm/logging/monitor/collector/manager'
-
-describe ScoutApm::Logging::Collector::Manager do
-  it 'should recreate the monitor process if monitor.pid file is errant' do
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    ScoutApm::Logging::Utils.ensure_directory_exists('/tmp/scout_apm/scout_apm_log_monitor.pid')
-
-    # A high enough number to not be a PID of a running process
-    pid_file_path = '/tmp/scout_apm/scout_apm_log_monitor.pid'
-    File.open(pid_file_path, 'w') do |file|
-      file.write('123456')
-    end
-
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
-    ScoutApm::Logging::MonitorManager.instance.setup!
-    ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
-
-    # Give the process time to initialize, download the collector, and start it
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    new_pid = File.read(pid_file_path).to_i
-
-    expect(new_pid).not_to eq(12_345)
-
-    # Check if the process with the stored PID is running
-    expect(Process.kill(0, new_pid)).to be_truthy
-    ENV.delete('SCOUT_LOGS_MONITOR')
-
-    # Kill the process and ensure PID file clean up
-    Process.kill('TERM', new_pid)
-    Process.kill('TERM', `pgrep otelcol-contrib --runstates D,R,S`.to_i)
-    sleep 1 # Give the process time to exit
-  end
-end

data/spec/integration/monitor_manager/single_monitor_spec.rb

@@ -1,53 +0,0 @@
-# This is really meant to simulate multiple processes calling the MonitorManager setup.
-# Trying to create **multiple fake** rails environments is a bit challening
-# (such as a rails app and a couple rails runners).
-
-# See: https://github.com/rails/rails/blob/6d126e03dbbf5d30fa97b580f7dee46343537b7b/railties/test/isolation/abstract_unit.rb#L339
-
-# We simulate the ultimate outcome here -- ie Railtie invoking the setup.
-require 'spec_helper'
-
-describe ScoutApm::Logging do
-  it 'Should only create a single monitor daemon if manager is called multiple times' do
-    ENV['SCOUT_LOGS_MONITOR'] = 'true'
-    ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
-
-    pid_file = ScoutApm::Logging::MonitorManager.instance.context.config.value('monitor_pid_file')
-    expect(File.exist?(pid_file)).to be_falsey
-
-    context = ScoutApm::Logging::MonitorManager.instance.context
-
-    pids = 3.times.map do
-      fork do
-        puts 'fork attempting to gain lock'
-        ScoutApm::Logging::Utils.attempt_exclusive_lock(context) do
-          puts 'obtained lock'
-          ScoutApm::Logging::MonitorManager.new.setup!
-        end
-      end
-    end
-
-    pids.each { |pid| Process.wait(pid) }
-
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    # The file lock really should be gone at this point.
-    expect(File.exist?(context.config.value('manager_lock_file'))).to be_falsey
-    expect(File.exist?(pid_file)).to be_truthy
-
-    original_pid = File.read(pid_file).to_i
-    expect(ScoutApm::Logging::Utils.check_process_liveliness(original_pid, 'scout_apm_logging_monitor')).to be_truthy
-
-    # Another process comes in and tries to start it again
-    ScoutApm::Logging::Utils.attempt_exclusive_lock(context) do
-      puts 'obtained lock later on'
-      ScoutApm::Logging::MonitorManager.new.setup!
-    end
-
-    expect(File.exist?(context.config.value('manager_lock_file'))).to be_falsey
-    expect(File.exist?(pid_file)).to be_truthy
-    updated_pid = File.read(pid_file).to_i
-    expect(updated_pid).to eq(original_pid)
-    expect(ScoutApm::Logging::Utils.check_process_liveliness(original_pid, 'scout_apm_logging_monitor')).to be_truthy
-  end
-end

data/spec/unit/monitor/collector/configuration_spec.rb

@@ -1,64 +0,0 @@
-require 'yaml'
-
-require 'spec_helper'
-
-require_relative '../../../../lib/scout_apm/logging/monitor/collector/configuration'
-
-describe ScoutApm::Logging::Collector::Configuration do
-  it 'creates the correct configuration for the otelcol' do
-    context = ScoutApm::Logging::Context.new
-    setup_collector_config!(context)
-
-    expect(File.exist?(context.config.value('collector_config_file'))).to be_truthy
-    config = YAML.load_file(context.config.value('collector_config_file'))
-
-    expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.scoutotel.com:4317')
-    expect(config['exporters']['otlp']['headers']['x-telemetryhub-key']).to eq('00001000010000abc')
-    expect(config['receivers']['filelog']['include']).to eq(['/tmp/fake_log_file.log'])
-  end
-
-  it 'merges in a logs config correctly' do
-    ENV['SCOUT_LOGS_CONFIG'] = File.expand_path('../../../data/logs_config.yml', __dir__)
-
-    context = ScoutApm::Logging::Context.new
-    setup_collector_config!(context)
-
-    expect(File.exist?(context.config.value('collector_config_file'))).to be_truthy
-    config = YAML.load_file(context.config.value('collector_config_file'))
-
-    expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.scoutotel.com:4317')
-    expect(config['exporters']['otlp']['headers']['x-telemetryhub-key']).to eq('00001000010000abc')
-    expect(config['receivers']['filelog']['include']).to eq(['/tmp/fake_log_file.log'])
-
-    # Verify merge and consistent keys
-    expect(config['extensions']['file_storage/otc']['directory']).to eq('/dev/null')
-    expect(config['extensions']['file_storage/otc']['timeout']).to eq('10s')
-  end
-
-  it 'handles an empty logs config file well' do
-    ENV['SCOUT_LOGS_CONFIG'] = File.expand_path('../../../data/empty_logs_config.yml', __dir__)
-
-    context = ScoutApm::Logging::Context.new
-    setup_collector_config!(context)
-
-    expect(File.exist?(context.config.value('collector_config_file'))).to be_truthy
-    config = YAML.load_file(context.config.value('collector_config_file'))
-
-    expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.scoutotel.com:4317')
-    expect(config['exporters']['otlp']['headers']['x-telemetryhub-key']).to eq('00001000010000abc')
-    expect(config['receivers']['filelog']['include']).to eq(['/tmp/fake_log_file.log'])
-  end
-
-  private
-
-  def setup_collector_config!(context)
-    ENV['SCOUT_MONITOR_STATE_FILE'] = File.expand_path('../../../data/state_file.json', __dir__)
-    conf_file = File.expand_path('../../../data/mock_config.yml', __dir__)
-    context.config = ScoutApm::Logging::Config.with_file(context, conf_file)
-
-    ScoutApm::Logging::Utils.ensure_directory_exists(context.config.value('collector_config_file'))
-
-    collector_config = ScoutApm::Logging::Collector::Configuration.new(context)
-    collector_config.setup!
-  end
-end

data/spec/unit/state_spec.rb

@@ -1,20 +0,0 @@
-require 'spec_helper'
-
-describe ScoutApm::Logging::Config do
-  it 'flushes and loads the state' do
-    context = ScoutApm::Logging::Context.new
-    conf_file = File.expand_path('../data/config_test_1.yml', __dir__)
-    conf = ScoutApm::Logging::Config.with_file(context, conf_file)
-
-    context.config = conf
-
-    ScoutApm::Logging::Config::ConfigDynamic.set_value('health_check_port', 1234)
-
-    context.config.state.flush_state!
-
-    data = ScoutApm::Logging::Config::State.new(context).load_state_from_file
-
-    expect(data['health_check_port']).to eq(context.config.value('health_check_port'))
-    expect(data['logs_monitored']).to eq(context.config.value('logs_monitored'))
-  end
-end

data/tooling/checksums.rb

@@ -1,106 +0,0 @@
-###
-# Calculates the checksums for the otelcol-contrib files
-# Usage: ruby tooling/checksums.rb <version>
-##
-# frozen_string_literal: true
-
-require 'digest'
-require 'fileutils'
-require 'net/http'
-require 'pp'
-
-DOUBLES = [
-  'darwin_amd64',
-  'darwin_arm64',
-  'linux_amd64',
-  'linux_arm64'
-]
-
-version = ARGV[0]
-
-current_directory = Dir.pwd
-unless current_directory.include?("/tooling")
-  tooling_directory = File.join(current_directory, "/tooling")
-  FileUtils.cd(tooling_directory)
-end
-
-def download_source_checksums(url, destination)
-  uri = URI(url)
-
-  Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
-    request = Net::HTTP::Get.new(uri)
-    http.request(request) do |response|
-      return download_source_checksums(response['location'], destination) if response.code == '302'
-
-      File.open(destination, 'wb') do |file|
-        response.read_body do |chunk|
-          file.write(chunk)
-        end
-      end
-    end
-  end
-end
-
-def validate_and_return_checksums(version)
-  checksum_file_url = "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v#{version}/opentelemetry-collector-releases_otelcol-contrib_checksums.txt"
-  destination = "./opentelemetry-collector-releases_otelcol-contrib_checksums.txt"
-
-  download_source_checksums(checksum_file_url, destination)
-
-  contents = File.read(destination)
-  lines = contents.split("\n")
-  pairs = lines.each_with_object({}) do |item, hash|
-    value, key = item.split
-    hash[key] = value
-  end
-
-
-  relavent_pairs = {}
-  DOUBLES.each do |double|
-    file_name = "otelcol-contrib_#{version}_#{double}.tar.gz"
-    file_location = "./#{file_name}"
-
-    local_checksum = Digest::SHA256.file(file_location).hexdigest
-    source_checksum = pairs[file_name]
-
-    if source_checksum != local_checksum
-      puts "#{double} contains different checksums: Local checksum #{local_checksum} -- Source checksum #{source_checksum}"
-      raise "Different checksums for #{double}"
-    end
-
-    relavent_pairs[double] = local_checksum
-  end
-
-  relavent_pairs
-end
-
-def download_collector(url, destination)
-  uri = URI(url)
-
-  Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
-    request = Net::HTTP::Get.new(uri)
-    http.request(request) do |response|
-      return download_collector(response['location'], destination) if response.code == '302'
-
-      File.open(destination, 'wb') do |file|
-        response.read_body do |chunk|
-          file.write(chunk)
-        end
-      end
-    end
-  end
-end
-
-DOUBLES.each do |double|
-  host_os, architecture = double.split('_')
-
-  url = "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v#{version}/otelcol-contrib_#{version}_#{host_os}_#{architecture}.tar.gz"
-
-  destination = "./otelcol-contrib_#{version}_#{host_os}_#{architecture}.tar.gz"
-
-  download_collector(url, destination)
-end
-
-checksums = validate_and_return_checksums(version)
-
-pp checksums