scout_apm_logging 0.0.13 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +0 -4
  3. data/.rubocop.yml +1 -1
  4. data/CHANGELOG.md +6 -0
  5. data/NOTICE +4 -0
  6. data/lib/scout_apm/logging/config.rb +5 -131
  7. data/lib/scout_apm/logging/loggers/capture.rb +3 -2
  8. data/lib/scout_apm/logging/loggers/formatter.rb +21 -2
  9. data/lib/scout_apm/logging/loggers/opentelemetry/LICENSE +201 -0
  10. data/lib/scout_apm/logging/loggers/opentelemetry/NOTICE +9 -0
  11. data/lib/scout_apm/logging/loggers/opentelemetry/api/logs/log_record.rb +18 -0
  12. data/lib/scout_apm/logging/loggers/opentelemetry/api/logs/logger.rb +64 -0
  13. data/lib/scout_apm/logging/loggers/opentelemetry/api/logs/logger_provider.rb +31 -0
  14. data/lib/scout_apm/logging/loggers/opentelemetry/api/logs/severity_number.rb +43 -0
  15. data/lib/scout_apm/logging/loggers/opentelemetry/api/logs/version.rb +18 -0
  16. data/lib/scout_apm/logging/loggers/opentelemetry/api/logs.rb +28 -0
  17. data/lib/scout_apm/logging/loggers/opentelemetry/exporter/exporter/otlp/logs_exporter.rb +389 -0
  18. data/lib/scout_apm/logging/loggers/opentelemetry/exporter/exporter/otlp/version.rb +20 -0
  19. data/lib/scout_apm/logging/loggers/opentelemetry/exporter/proto/collector/logs/v1/logs_service_pb.rb +43 -0
  20. data/lib/scout_apm/logging/loggers/opentelemetry/exporter/proto/common/v1/common_pb.rb +58 -0
  21. data/lib/scout_apm/logging/loggers/opentelemetry/exporter/proto/logs/v1/logs_pb.rb +91 -0
  22. data/lib/scout_apm/logging/loggers/opentelemetry/exporter/proto/resource/v1/resource_pb.rb +33 -0
  23. data/lib/scout_apm/logging/loggers/opentelemetry/opentelemetry.rb +62 -0
  24. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/export/batch_log_record_processor.rb +225 -0
  25. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/export/log_record_exporter.rb +64 -0
  26. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/export.rb +34 -0
  27. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/log_record.rb +115 -0
  28. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/log_record_data.rb +31 -0
  29. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/log_record_processor.rb +53 -0
  30. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/logger.rb +94 -0
  31. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/logger_provider.rb +158 -0
  32. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/version.rb +20 -0
  33. data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs.rb +28 -0
  34. data/lib/scout_apm/logging/utils.rb +0 -69
  35. data/lib/scout_apm/logging/version.rb +1 -1
  36. data/lib/scout_apm_logging.rb +2 -11
  37. data/scout_apm_logging.gemspec +7 -0
  38. data/spec/data/config_test_1.yml +0 -1
  39. data/spec/data/mock_config.yml +0 -3
  40. data/spec/integration/rails/lifecycle_spec.rb +57 -23
  41. data/spec/spec_helper.rb +0 -12
  42. data/spec/unit/config_spec.rb +0 -12
  43. data/spec/unit/loggers/capture_spec.rb +0 -6
  44. metadata +126 -39
  45. data/bin/scout_apm_logging_monitor +0 -6
  46. data/lib/scout_apm/logging/monitor/_rails.rb +0 -22
  47. data/lib/scout_apm/logging/monitor/collector/checksum.rb +0 -51
  48. data/lib/scout_apm/logging/monitor/collector/configuration.rb +0 -150
  49. data/lib/scout_apm/logging/monitor/collector/downloader.rb +0 -78
  50. data/lib/scout_apm/logging/monitor/collector/extractor.rb +0 -37
  51. data/lib/scout_apm/logging/monitor/collector/manager.rb +0 -57
  52. data/lib/scout_apm/logging/monitor/monitor.rb +0 -216
  53. data/lib/scout_apm/logging/monitor_manager/manager.rb +0 -162
  54. data/lib/scout_apm/logging/state.rb +0 -69
  55. data/spec/data/empty_logs_config.yml +0 -0
  56. data/spec/data/logs_config.yml +0 -3
  57. data/spec/data/state_file.json +0 -3
  58. data/spec/integration/loggers/capture_spec.rb +0 -68
  59. data/spec/integration/monitor/collector/downloader/will_verify_checksum.rb +0 -49
  60. data/spec/integration/monitor/collector_healthcheck_spec.rb +0 -29
  61. data/spec/integration/monitor/continuous_state_collector_spec.rb +0 -31
  62. data/spec/integration/monitor/previous_collector_setup_spec.rb +0 -45
  63. data/spec/integration/monitor_manager/disable_agent_spec.rb +0 -30
  64. data/spec/integration/monitor_manager/monitor_pid_file_spec.rb +0 -38
  65. data/spec/integration/monitor_manager/single_monitor_spec.rb +0 -53
  66. data/spec/unit/monitor/collector/configuration_spec.rb +0 -64
  67. data/spec/unit/state_spec.rb +0 -20
  68. data/tooling/checksums.rb +0 -106
@@ -1,30 +0,0 @@
1
- require 'spec_helper'
2
-
3
- require_relative '../../../lib/scout_apm/logging/monitor/collector/manager'
4
-
5
- describe ScoutApm::Logging::Collector::Manager do
6
- it 'If monitor is false, it should remove the daemon and collector process if they are present' do
7
- ENV['SCOUT_LOGS_MONITOR'] = 'true'
8
- ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
9
-
10
- expect(`pgrep otelcol-contrib --runstates D,R,S`).to be_empty
11
-
12
- ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
13
- ScoutApm::Logging::MonitorManager.instance.setup!
14
- ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
15
-
16
- wait_for_process_with_timeout!('otelcol-contrib', 20)
17
-
18
- ENV['SCOUT_LOGS_MONITOR'] = 'false'
19
-
20
- ScoutApm::Logging::MonitorManager.new.setup!
21
-
22
- sleep 5 # Give the process time to exit
23
-
24
- expect(File.exist?(ScoutApm::Logging::MonitorManager.instance.context.config.value('monitor_pid_file'))).to be_falsey
25
- expect(`pgrep otelcol-contrib --runstates D,R,S`).to be_empty
26
- expect(`pgrep scout_apm_log_monitor --runstates D,R,S`).to be_empty
27
-
28
- ENV.delete('SCOUT_LOGS_MONITOR')
29
- end
30
- end
@@ -1,38 +0,0 @@
1
- require 'spec_helper'
2
-
3
- require_relative '../../../lib/scout_apm/logging/monitor/collector/manager'
4
-
5
- describe ScoutApm::Logging::Collector::Manager do
6
- it 'should recreate the monitor process if monitor.pid file is errant' do
7
- ENV['SCOUT_LOGS_MONITOR'] = 'true'
8
- ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
9
-
10
- ScoutApm::Logging::Utils.ensure_directory_exists('/tmp/scout_apm/scout_apm_log_monitor.pid')
11
-
12
- # A high enough number to not be a PID of a running process
13
- pid_file_path = '/tmp/scout_apm/scout_apm_log_monitor.pid'
14
- File.open(pid_file_path, 'w') do |file|
15
- file.write('123456')
16
- end
17
-
18
- ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time start: #{Time.now}"
19
- ScoutApm::Logging::MonitorManager.instance.setup!
20
- ScoutApm::Logging::MonitorManager.instance.context.logger.info "Time after setup: #{Time.now}"
21
-
22
- # Give the process time to initialize, download the collector, and start it
23
- wait_for_process_with_timeout!('otelcol-contrib', 20)
24
-
25
- new_pid = File.read(pid_file_path).to_i
26
-
27
- expect(new_pid).not_to eq(12_345)
28
-
29
- # Check if the process with the stored PID is running
30
- expect(Process.kill(0, new_pid)).to be_truthy
31
- ENV.delete('SCOUT_LOGS_MONITOR')
32
-
33
- # Kill the process and ensure PID file clean up
34
- Process.kill('TERM', new_pid)
35
- Process.kill('TERM', `pgrep otelcol-contrib --runstates D,R,S`.to_i)
36
- sleep 1 # Give the process time to exit
37
- end
38
- end
@@ -1,53 +0,0 @@
1
- # This is really meant to simulate multiple processes calling the MonitorManager setup.
2
- # Trying to create **multiple fake** rails environments is a bit challening
3
- # (such as a rails app and a couple rails runners).
4
-
5
- # See: https://github.com/rails/rails/blob/6d126e03dbbf5d30fa97b580f7dee46343537b7b/railties/test/isolation/abstract_unit.rb#L339
6
-
7
- # We simulate the ultimate outcome here -- ie Railtie invoking the setup.
8
- require 'spec_helper'
9
-
10
- describe ScoutApm::Logging do
11
- it 'Should only create a single monitor daemon if manager is called multiple times' do
12
- ENV['SCOUT_LOGS_MONITOR'] = 'true'
13
- ENV['SCOUT_LOGS_MONITORED'] = '["/tmp/test.log"]'
14
-
15
- pid_file = ScoutApm::Logging::MonitorManager.instance.context.config.value('monitor_pid_file')
16
- expect(File.exist?(pid_file)).to be_falsey
17
-
18
- context = ScoutApm::Logging::MonitorManager.instance.context
19
-
20
- pids = 3.times.map do
21
- fork do
22
- puts 'fork attempting to gain lock'
23
- ScoutApm::Logging::Utils.attempt_exclusive_lock(context) do
24
- puts 'obtained lock'
25
- ScoutApm::Logging::MonitorManager.new.setup!
26
- end
27
- end
28
- end
29
-
30
- pids.each { |pid| Process.wait(pid) }
31
-
32
- wait_for_process_with_timeout!('otelcol-contrib', 20)
33
-
34
- # The file lock really should be gone at this point.
35
- expect(File.exist?(context.config.value('manager_lock_file'))).to be_falsey
36
- expect(File.exist?(pid_file)).to be_truthy
37
-
38
- original_pid = File.read(pid_file).to_i
39
- expect(ScoutApm::Logging::Utils.check_process_liveliness(original_pid, 'scout_apm_logging_monitor')).to be_truthy
40
-
41
- # Another process comes in and tries to start it again
42
- ScoutApm::Logging::Utils.attempt_exclusive_lock(context) do
43
- puts 'obtained lock later on'
44
- ScoutApm::Logging::MonitorManager.new.setup!
45
- end
46
-
47
- expect(File.exist?(context.config.value('manager_lock_file'))).to be_falsey
48
- expect(File.exist?(pid_file)).to be_truthy
49
- updated_pid = File.read(pid_file).to_i
50
- expect(updated_pid).to eq(original_pid)
51
- expect(ScoutApm::Logging::Utils.check_process_liveliness(original_pid, 'scout_apm_logging_monitor')).to be_truthy
52
- end
53
- end
@@ -1,64 +0,0 @@
1
- require 'yaml'
2
-
3
- require 'spec_helper'
4
-
5
- require_relative '../../../../lib/scout_apm/logging/monitor/collector/configuration'
6
-
7
- describe ScoutApm::Logging::Collector::Configuration do
8
- it 'creates the correct configuration for the otelcol' do
9
- context = ScoutApm::Logging::Context.new
10
- setup_collector_config!(context)
11
-
12
- expect(File.exist?(context.config.value('collector_config_file'))).to be_truthy
13
- config = YAML.load_file(context.config.value('collector_config_file'))
14
-
15
- expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.scoutotel.com:4317')
16
- expect(config['exporters']['otlp']['headers']['x-telemetryhub-key']).to eq('00001000010000abc')
17
- expect(config['receivers']['filelog']['include']).to eq(['/tmp/fake_log_file.log'])
18
- end
19
-
20
- it 'merges in a logs config correctly' do
21
- ENV['SCOUT_LOGS_CONFIG'] = File.expand_path('../../../data/logs_config.yml', __dir__)
22
-
23
- context = ScoutApm::Logging::Context.new
24
- setup_collector_config!(context)
25
-
26
- expect(File.exist?(context.config.value('collector_config_file'))).to be_truthy
27
- config = YAML.load_file(context.config.value('collector_config_file'))
28
-
29
- expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.scoutotel.com:4317')
30
- expect(config['exporters']['otlp']['headers']['x-telemetryhub-key']).to eq('00001000010000abc')
31
- expect(config['receivers']['filelog']['include']).to eq(['/tmp/fake_log_file.log'])
32
-
33
- # Verify merge and consistent keys
34
- expect(config['extensions']['file_storage/otc']['directory']).to eq('/dev/null')
35
- expect(config['extensions']['file_storage/otc']['timeout']).to eq('10s')
36
- end
37
-
38
- it 'handles an empty logs config file well' do
39
- ENV['SCOUT_LOGS_CONFIG'] = File.expand_path('../../../data/empty_logs_config.yml', __dir__)
40
-
41
- context = ScoutApm::Logging::Context.new
42
- setup_collector_config!(context)
43
-
44
- expect(File.exist?(context.config.value('collector_config_file'))).to be_truthy
45
- config = YAML.load_file(context.config.value('collector_config_file'))
46
-
47
- expect(config['exporters']['otlp']['endpoint']).to eq('https://otlp.scoutotel.com:4317')
48
- expect(config['exporters']['otlp']['headers']['x-telemetryhub-key']).to eq('00001000010000abc')
49
- expect(config['receivers']['filelog']['include']).to eq(['/tmp/fake_log_file.log'])
50
- end
51
-
52
- private
53
-
54
- def setup_collector_config!(context)
55
- ENV['SCOUT_MONITOR_STATE_FILE'] = File.expand_path('../../../data/state_file.json', __dir__)
56
- conf_file = File.expand_path('../../../data/mock_config.yml', __dir__)
57
- context.config = ScoutApm::Logging::Config.with_file(context, conf_file)
58
-
59
- ScoutApm::Logging::Utils.ensure_directory_exists(context.config.value('collector_config_file'))
60
-
61
- collector_config = ScoutApm::Logging::Collector::Configuration.new(context)
62
- collector_config.setup!
63
- end
64
- end
@@ -1,20 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe ScoutApm::Logging::Config do
4
- it 'flushes and loads the state' do
5
- context = ScoutApm::Logging::Context.new
6
- conf_file = File.expand_path('../data/config_test_1.yml', __dir__)
7
- conf = ScoutApm::Logging::Config.with_file(context, conf_file)
8
-
9
- context.config = conf
10
-
11
- ScoutApm::Logging::Config::ConfigDynamic.set_value('health_check_port', 1234)
12
-
13
- context.config.state.flush_state!
14
-
15
- data = ScoutApm::Logging::Config::State.new(context).load_state_from_file
16
-
17
- expect(data['health_check_port']).to eq(context.config.value('health_check_port'))
18
- expect(data['logs_monitored']).to eq(context.config.value('logs_monitored'))
19
- end
20
- end
data/tooling/checksums.rb DELETED
@@ -1,106 +0,0 @@
1
- ###
2
- # Calculates the checksums for the otelcol-contrib files
3
- # Usage: ruby tooling/checksums.rb <version>
4
- ##
5
- # frozen_string_literal: true
6
-
7
- require 'digest'
8
- require 'fileutils'
9
- require 'net/http'
10
- require 'pp'
11
-
12
- DOUBLES = [
13
- 'darwin_amd64',
14
- 'darwin_arm64',
15
- 'linux_amd64',
16
- 'linux_arm64'
17
- ]
18
-
19
- version = ARGV[0]
20
-
21
- current_directory = Dir.pwd
22
- unless current_directory.include?("/tooling")
23
- tooling_directory = File.join(current_directory, "/tooling")
24
- FileUtils.cd(tooling_directory)
25
- end
26
-
27
- def download_source_checksums(url, destination)
28
- uri = URI(url)
29
-
30
- Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
31
- request = Net::HTTP::Get.new(uri)
32
- http.request(request) do |response|
33
- return download_source_checksums(response['location'], destination) if response.code == '302'
34
-
35
- File.open(destination, 'wb') do |file|
36
- response.read_body do |chunk|
37
- file.write(chunk)
38
- end
39
- end
40
- end
41
- end
42
- end
43
-
44
- def validate_and_return_checksums(version)
45
- checksum_file_url = "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v#{version}/opentelemetry-collector-releases_otelcol-contrib_checksums.txt"
46
- destination = "./opentelemetry-collector-releases_otelcol-contrib_checksums.txt"
47
-
48
- download_source_checksums(checksum_file_url, destination)
49
-
50
- contents = File.read(destination)
51
- lines = contents.split("\n")
52
- pairs = lines.each_with_object({}) do |item, hash|
53
- value, key = item.split
54
- hash[key] = value
55
- end
56
-
57
-
58
- relavent_pairs = {}
59
- DOUBLES.each do |double|
60
- file_name = "otelcol-contrib_#{version}_#{double}.tar.gz"
61
- file_location = "./#{file_name}"
62
-
63
- local_checksum = Digest::SHA256.file(file_location).hexdigest
64
- source_checksum = pairs[file_name]
65
-
66
- if source_checksum != local_checksum
67
- puts "#{double} contains different checksums: Local checksum #{local_checksum} -- Source checksum #{source_checksum}"
68
- raise "Different checksums for #{double}"
69
- end
70
-
71
- relavent_pairs[double] = local_checksum
72
- end
73
-
74
- relavent_pairs
75
- end
76
-
77
- def download_collector(url, destination)
78
- uri = URI(url)
79
-
80
- Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
81
- request = Net::HTTP::Get.new(uri)
82
- http.request(request) do |response|
83
- return download_collector(response['location'], destination) if response.code == '302'
84
-
85
- File.open(destination, 'wb') do |file|
86
- response.read_body do |chunk|
87
- file.write(chunk)
88
- end
89
- end
90
- end
91
- end
92
- end
93
-
94
- DOUBLES.each do |double|
95
- host_os, architecture = double.split('_')
96
-
97
- url = "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v#{version}/otelcol-contrib_#{version}_#{host_os}_#{architecture}.tar.gz"
98
-
99
- destination = "./otelcol-contrib_#{version}_#{host_os}_#{architecture}.tar.gz"
100
-
101
- download_collector(url, destination)
102
- end
103
-
104
- checksums = validate_and_return_checksums(version)
105
-
106
- pp checksums