scout_apm_logging 0.0.13 → 1.0.0

data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/logger.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# Copyright The OpenTelemetry Authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+module ScoutApm
+  module Logging
+    module Loggers
+      module OpenTelemetry
+        module SDK
+          module Logs
+            # The SDK implementation of OpenTelemetry::Logs::Logger
+            class Logger < OpenTelemetry::Logs::Logger
+              # @api private
+              #
+              # Returns a new {OpenTelemetry::SDK::Logs::Logger} instance. This should
+              # not be called directly. New loggers should be created using
+              # {LoggerProvider#logger}.
+              #
+              # @param [String] name Instrumentation package name
+              # @param [String] version Instrumentation package version
+              # @param [LoggerProvider] logger_provider The {LoggerProvider} that
+              #   initialized the logger
+              #
+              # @return [OpenTelemetry::SDK::Logs::Logger]
+              def initialize(name, version, logger_provider)
+                @instrumentation_scope = ::OpenTelemetry::SDK::InstrumentationScope.new(name, version)
+                @logger_provider = logger_provider
+              end
+
+              # Emit a {LogRecord} to the processing pipeline.
+              #
+              # @param [optional Time] timestamp Time when the event occurred.
+              # @param [optional Time] observed_timestamp Time when the event was
+              #   observed by the collection system.
+              # @param [optional OpenTelemetry::Trace::SpanContext] span_context The
+              #   OpenTelemetry::Trace::SpanContext to associate with the
+              #   {LogRecord}.
+              # @param severity_number [optional Integer] Numerical value of the
+              #   severity. Smaller numerical values correspond to less severe events
+              #   (such as debug events), larger numerical values correspond to more
+              #   severe events (such as errors and critical events).
+              # @param [optional String, Numeric, Boolean, Array<String, Numeric,
+              #   Boolean>, Hash{String => String, Numeric, Boolean, Array<String,
+              #   Numeric, Boolean>}] body A value containing the body of the log record.
+              # @param [optional Hash{String => String, Numeric, Boolean,
+              #   Array<String, Numeric, Boolean>}] attributes Additional information
+              #   about the event.
+              # @param [optional String (16-byte binary)] trace_id Request trace id as
+              #   defined in {https://www.w3.org/TR/trace-context/#trace-id W3C Trace Context}.
+              #   Can be set for logs that are part of request processing and have an
+              #   assigned trace id.
+              # @param [optional String (8-byte binary)] span_id Span id. Can be set
+              #   for logs that are part of a particular processing span. If span_id
+              #   is present trace_id should also be present.
+              # @param [optional Integer (8-bit byte of bit flags)] trace_flags Trace
+              #   flag as defined in {https://www.w3.org/TR/trace-context/#trace-flags W3C Trace Context}
+              #   specification. At the time of writing the specification defines one
+              #   flag - the SAMPLED flag.
+              # @param [optional OpenTelemetry::Context] context The OpenTelemetry::Context
+              #   to associate with the {LogRecord}.
+              #
+              # @api public
+              def on_emit(timestamp: nil,
+                          observed_timestamp: Time.now,
+                          severity_text: nil,
+                          severity_number: nil,
+                          body: nil,
+                          attributes: nil,
+                          trace_id: nil,
+                          span_id: nil,
+                          trace_flags: nil,
+                          context: ::OpenTelemetry::Context.current)
+
+                @logger_provider.on_emit(timestamp: timestamp,
+                                        observed_timestamp: observed_timestamp,
+                                        severity_text: severity_text,
+                                        severity_number: severity_number,
+                                        body: body,
+                                        attributes: attributes,
+                                        trace_id: nil,
+                                        span_id: nil,
+                                        trace_flags: nil,
+                                        instrumentation_scope: @instrumentation_scope,
+                                        context: context)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/logger_provider.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+# Copyright The OpenTelemetry Authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+module ScoutApm
+  module Logging
+    module Loggers
+      module OpenTelemetry
+        module SDK
+          module Logs
+            # The SDK implementation of OpenTelemetry::Logs::LoggerProvider.
+            class LoggerProvider < OpenTelemetry::Logs::LoggerProvider
+              UNEXPECTED_ERROR_MESSAGE = 'unexpected error in ' \
+                'OpenTelemetry::SDK::Logs::LoggerProvider#%s'
+
+              private_constant :UNEXPECTED_ERROR_MESSAGE
+
+              # Returns a new LoggerProvider instance.
+              #
+              # @param [optional Resource] resource The resource to associate with
+              #   new LogRecords created by {Logger}s created by this LoggerProvider.
+              #
+              # @return [OpenTelemetry::SDK::Logs::LoggerProvider]
+              def initialize(resource: OpenTelemetry::SDK::Resources::Resource.create)
+                @log_record_processors = []
+                @mutex = Mutex.new
+                @resource = resource
+                @stopped = false
+              end
+
+              # Returns an {OpenTelemetry::SDK::Logs::Logger} instance.
+              #
+              # @param [String] name Instrumentation package name
+              # @param [optional String] version Instrumentation package version
+              #
+              # @return [OpenTelemetry::SDK::Logs::Logger]
+              def logger(name:, version: nil)
+                version ||= ''
+
+                if !name.is_a?(String) || name.empty?
+                  OpenTelemetry.logger.warn('LoggerProvider#logger called with an ' \
+                    "invalid name. Name provided: #{name.inspect}")
+                end
+
+                Logger.new(name, version, self)
+              end
+
+              # Adds a new log record processor to this LoggerProvider's
+              # log_record_processors.
+              #
+              # @param [LogRecordProcessor] log_record_processor The
+              #   {LogRecordProcessor} to add to this LoggerProvider.
+              def add_log_record_processor(log_record_processor)
+                @mutex.synchronize do
+                  if @stopped
+                    OpenTelemetry.logger.warn('calling LoggerProvider#' \
+                      'add_log_record_processor after shutdown.')
+                    return
+                  end
+                  @log_record_processors = @log_record_processors.dup.push(log_record_processor)
+                end
+              end
+
+              # Attempts to stop all the activity for this LoggerProvider. Calls
+              # {LogRecordProcessor#shutdown} for all registered {LogRecordProcessor}s.
+              #
+              # This operation may block until all log records are processed. Must
+              # be called before turning off the main application to ensure all data
+              # are processed and exported.
+              #
+              # After this is called all newly created {LogRecord}s will be no-op.
+              #
+              # @param [optional Numeric] timeout An optional timeout in seconds.
+              # @return [Integer] Export::SUCCESS if no error occurred, Export::FAILURE if
+              #   a non-specific failure occurred, Export::TIMEOUT if a timeout occurred.
+              def shutdown(timeout: nil)
+                @mutex.synchronize do
+                  if @stopped
+                    OpenTelemetry.logger.warn('LoggerProvider#shutdown called multiple times.')
+                    return Export::FAILURE
+                  end
+
+                  start_time = OpenTelemetry::Common::Utilities.timeout_timestamp
+                  results = @log_record_processors.map do |processor|
+                    remaining_timeout = OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time)
+                    break [Export::TIMEOUT] if remaining_timeout&.zero?
+
+                    processor.shutdown(timeout: remaining_timeout)
+                  end
+
+                  @stopped = true
+                  results.max || Export::SUCCESS
+                end
+              end
+
+              # Immediately export all {LogRecord}s that have not yet been exported
+              # for all the registered {LogRecordProcessor}s.
+              #
+              # This method should only be called in cases where it is absolutely
+              # necessary, such as when using some FaaS providers that may suspend
+              # the process after an invocation, but before the {LogRecordProcessor}
+              # exports the completed {LogRecord}s.
+              #
+              # @param [optional Numeric] timeout An optional timeout in seconds.
+              # @return [Integer] Export::SUCCESS if no error occurred, Export::FAILURE if
+              #   a non-specific failure occurred, Export::TIMEOUT if a timeout occurred.
+              def force_flush(timeout: nil)
+                @mutex.synchronize do
+                  return Export::SUCCESS if @stopped
+
+                  start_time = OpenTelemetry::Common::Utilities.timeout_timestamp
+                  results = @log_record_processors.map do |processor|
+                    remaining_timeout = OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time)
+                    return Export::TIMEOUT if remaining_timeout&.zero?
+
+                    processor.force_flush(timeout: remaining_timeout)
+                  end
+
+                  results.max || Export::SUCCESS
+                end
+              end
+
+              # @api private
+              def on_emit(timestamp: nil,
+                          observed_timestamp: nil,
+                          severity_text: nil,
+                          severity_number: nil,
+                          body: nil,
+                          attributes: nil,
+                          trace_id: nil,
+                          span_id: nil,
+                          trace_flags: nil,
+                          instrumentation_scope: nil,
+                          context: nil)
+
+                log_record = LogRecord.new(timestamp: timestamp,
+                                          observed_timestamp: observed_timestamp,
+                                          severity_text: severity_text,
+                                          severity_number: severity_number,
+                                          body: body,
+                                          attributes: attributes,
+                                          trace_id: trace_id,
+                                          span_id: span_id,
+                                          trace_flags: trace_flags,
+                                          resource: @resource,
+                                          instrumentation_scope: instrumentation_scope)
+
+                @log_record_processors.each { |processor| processor.on_emit(log_record, context) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/version.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright The OpenTelemetry Authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+module ScoutApm
+  module Logging
+    module Loggers
+      module OpenTelemetry
+        module SDK
+          module Logs
+            # Current OpenTelemetry logs sdk version
+            VERSION = '0.1.0'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/loggers/opentelemetry/sdk/logs.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# Copyright The OpenTelemetry Authors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+require_relative 'logs/version'
+require_relative 'logs/logger'
+require_relative 'logs/logger_provider'
+require_relative 'logs/log_record'
+require_relative 'logs/log_record_data'
+require_relative 'logs/log_record_processor'
+require_relative 'logs/export'
+
+module ScoutApm
+  module Logging
+    module Loggers
+      module OpenTelemetry
+        module SDK
+          # The Logs module contains the OpenTelemetry logs reference
+          # implementation.
+          module Logs
+          end
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/logging/utils.rb

@@ -12,75 +12,6 @@ module ScoutApm
 
         FileUtils.mkdir_p(file_path) unless File.directory?(file_path)
       end
-
-      # TODO: Add support for other platforms
-      def self.get_architecture
-        if /arm/ =~ RbConfig::CONFIG['arch']
-          'arm64'
-        else
-          'amd64'
-        end
-      end
-
-      def self.get_host_os
-        if /darwin|mac os/ =~ RbConfig::CONFIG['host_os']
-          'darwin'
-        else
-          'linux'
-        end
-      end
-
-      def self.check_process_liveliness(pid, name)
-        # Pipe to cat to prevent truncation of the output
-        process_information = `ps -p #{pid} -o pid=,stat=,command= | cat`
-        return false if process_information.empty?
-
-        process_information_parts = process_information.split(' ')
-        process_information_status = process_information_parts[1]
-
-        return false if process_information_status == 'Z'
-        return false unless process_information.include?(name)
-
-        true
-      end
-
-      def self.current_process_is_app_server?
-        # TODO: Add more app servers.
-        process_command = `ps -p #{Process.pid} -o command= | cat`.downcase
-        [
-          process_command.include?('puma'),
-          process_command.include?('unicorn'),
-          process_command.include?('passenger')
-        ].any?
-      end
-
-      def self.skip_setup?
-        [
-          ARGV.include?('assets:precompile'),
-          ARGV.include?('assets:clean'),
-          (defined?(::Rails::Console) && $stdout.isatty && $stdin.isatty)
-        ].any?
-      end
-
-      def self.attempt_exclusive_lock(context)
-        lock_file = context.config.value('manager_lock_file')
-        ensure_directory_exists(lock_file)
-
-        begin
-          file = File.open(lock_file, File::RDWR | File::CREAT | File::EXCL)
-        rescue Errno::EEXIST
-          context.logger.info('Manager lock file held, continuing.')
-          return
-        end
-
-        # Ensure the lock file is deleted when the block completes
-        begin
-          yield
-        ensure
-          file.close
-          File.delete(lock_file) if File.exist?(lock_file)
-        end
-      end
     end
   end
 end

data/lib/scout_apm/logging/version.rb

@@ -2,6 +2,6 @@
 
 module ScoutApm
   module Logging
-    VERSION = '0.0.13'
+    VERSION = '1.0.0'
   end
 end

data/lib/scout_apm_logging.rb

@@ -6,12 +6,8 @@ require 'scout_apm/logging/config'
 require 'scout_apm/logging/logger'
 require 'scout_apm/logging/context'
 require 'scout_apm/logging/utils'
-require 'scout_apm/logging/state'
-
 require 'scout_apm/logging/loggers/capture'
 
-require 'scout_apm/logging/monitor_manager/manager'
-
 module ScoutApm
   ## This module is responsible for setting up monitoring of the application's logs.
   module Logging
@@ -19,15 +15,10 @@ module ScoutApm
       # If we are in a Rails environment, setup the monitor daemon manager.
       class RailTie < ::Rails::Railtie
         initializer 'scout_apm_logging.monitor', after: :initialize_logger, before: :initialize_cache do
-          context = ScoutApm::Logging::MonitorManager.instance.context
+          context = Context.new
+          context.config = Config.with_file(context, context.config.value('config_file'))
 
           Loggers::Capture.new(context).setup!
-
-          unless Utils.skip_setup?
-            Utils.attempt_exclusive_lock(context) do
-              ScoutApm::Logging::MonitorManager.instance.setup!
-            end
-          end
         end
       end
     end

data/scout_apm_logging.gemspec

@@ -17,9 +17,16 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = '>= 2.6'
 
+  s.add_dependency 'googleapis-common-protos-types'
+  s.add_dependency 'google-protobuf', '< 3.18'
+  s.add_dependency 'opentelemetry-api'
+  s.add_dependency 'opentelemetry-common'
+  s.add_dependency 'opentelemetry-instrumentation-base'
+  s.add_dependency 'opentelemetry-sdk', '>= 1.2'
   s.add_dependency 'scout_apm'
 
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'rubocop', '1.50.2'
   s.add_development_dependency 'rubocop-ast', '1.30.0'
+  s.add_development_dependency 'webmock'
 end

data/spec/data/config_test_1.yml

@@ -9,7 +9,6 @@ common: &defaults
   ###
   logs_monitor: true
   logs_ingest_key: "00001000010000abc"
-  logs_monitored: ["/tmp/fake_log_file.log"]
 
 production:
   <<: *defaults

data/spec/data/mock_config.yml

@@ -9,9 +9,6 @@ common: &defaults
   ###
   logs_monitor: true
   logs_ingest_key: "00001000010000abc"
-  logs_monitored: ["/tmp/fake_log_file.log"]
-  # Need to give a high enough number for the original health check to pass
-  monitor_interval: 10
 
 production:
   <<: *defaults

data/spec/integration/rails/lifecycle_spec.rb

@@ -1,32 +1,45 @@
+require 'webmock/rspec'
+
 require 'spec_helper'
+require 'zlib'
+require 'stringio'
 require_relative '../../rails/app'
 
 describe ScoutApm::Logging do
+  before do
+    @file_path = '/app/response_body.txt'
+    # Capture the outgoing HTTP request to inspect its values
+    stub_request(:post, 'https://otlp-devel.scoutotel.com:4318/v1/logs')
+      .to_return do |request|
+        # We have to write to the file, as we are applying the patch in the fork, and
+        # need to run the expectations in the main process.
+        File.open(@file_path, 'a+') do |file|
+          msgs = transform_body_to_msgs(request.body)
+          file.puts(msgs)
+        end
+        { body: '', headers: {}, status: 200 }
+      end
+  end
+
   it 'checks the Rails lifecycle for creating the daemon and collector processes' do
     ENV['SCOUT_LOGS_MONITOR'] = 'true'
+    ENV['SCOUT_LOGS_REPORTING_ENDPOINT_HTTP'] = 'https://otlp-devel.scoutotel.com:4318/v1/logs'
 
-    context = ScoutApm::Logging::MonitorManager.instance.context
-    pid_file = context.config.value('monitor_pid_file')
-    expect(File.exist?(pid_file)).to be_falsey
+    context = ScoutApm::Logging::Context.new
+    context.config = ScoutApm::Logging::Config.with_file(context, context.config.value('config_file'))
 
     rails_pid = fork do
       initialize_app
     end
 
-    wait_for_process_with_timeout!('otelcol-contrib', 20)
-
-    # Check if the monitor PID file exists
-    expect(File.exist?(pid_file)).to be_truthy
-
-    # Read the PID from the PID file
-    pid = File.read(pid_file).to_i
-
-    # Check if the process with the stored PID is running
-    expect(ScoutApm::Logging::Utils.check_process_liveliness(pid, 'scout_apm_logging_monitor')).to be_truthy
+    # TODO: Improve check to wait for Rails initialization.
+    sleep 5
 
     # Call the app to generate the logs
     `curl localhost:8080`
 
+    sleep 5
+
     proxy_dir = context.config.value('logs_proxy_log_dir')
     files = Dir.entries(proxy_dir) - ['.', '..']
     log_file = File.join(proxy_dir, files[0])
@@ -41,24 +54,45 @@ describe ScoutApm::Logging do
       end
     end
 
-    messages = lines.map { |item| item['msg'] }
+    local_messages = lines.map { |item| item['msg'] }
 
-    # Verify we have all the logs
-    expect(messages.count('[TEST] Some log')).to eq(1)
-    expect(messages.count('[YIELD] Yield Test')).to eq(1)
-    expect(messages.count('Another Log')).to eq(1)
-    expect(messages.count('Should not be captured')).to eq(0)
+    # Verify we have all the logs in the local log file
+    expect(local_messages.count('[TEST] Some log')).to eq(1)
+    expect(local_messages.count('[YIELD] Yield Test')).to eq(1)
+    expect(local_messages.count('Another Log')).to eq(1)
+    expect(local_messages.count('Should not be captured')).to eq(0)
 
     log_locations = lines.map { |item| item['log_location'] }.compact
 
     # Verify that log attributes aren't persisted
     expect(log_locations.size).to eq(1)
 
+    # Verify the logs are sent to the receiver
+    receiver_contents = File.readlines(@file_path, chomp: true)
+    expect(receiver_contents.count('[TEST] Some log')).to eq(1)
+    expect(receiver_contents.count('[YIELD] Yield Test')).to eq(1)
+    expect(receiver_contents.count('Another Log')).to eq(1)
+    expect(receiver_contents.count('Should not be captured')).to eq(0)
+
     # Kill the rails process. We use kill as using any other signal throws a long log line.
     Process.kill('KILL', rails_pid)
-    # Kill the process and ensure PID file clean up
-    Process.kill('TERM', pid)
-    sleep 1 # Give the process time to exit
-    expect(File.exist?(pid_file)).to be_falsey
+  end
+
+  private
+
+  def transform_body_to_msgs(body)
+    gz = Zlib::GzipReader.new(StringIO.new(body))
+    uncompressed = gz.read
+
+    value = ScoutApm::Logging::Loggers::Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode(uncompressed)
+    value_hash = value.to_h
+
+    value_hash[:resource_logs].map do |item|
+      item[:scope_logs].map do |sl|
+        sl[:log_records].map do |log|
+          log[:body][:string_value]
+        end
+      end
+    end.flatten
   end
 end

data/spec/spec_helper.rb

@@ -23,20 +23,8 @@ end
 RSpec.configure do |config|
   ENV["SCOUT_LOG_FILE_PATH"] = "STDOUT"
   ENV["SCOUT_LOG_LEVEL"] = "debug"
-  ENV["SCOUT_COLLECTOR_LOG_LEVEL"] = "info"
 
   config.after(:each) do
     RSpec::Mocks.space.reset_all
   end
 end
-
-def wait_for_process_with_timeout!(name, timeout_time)
-  Timeout::timeout(timeout_time) do
-    loop do
-      break if `pgrep #{name} --runstates D,R,S`.strip != ""
-      sleep 0.1
-    end
-  end
-
-  sleep 1
-end

data/spec/unit/config_spec.rb

@@ -8,18 +8,6 @@ describe ScoutApm::Logging::Config do
 
     expect(conf.value('log_level')).to eq('debug')
     expect(conf.value('logs_monitor')).to eq(true)
-    expect(conf.value('monitor_pid_file')).to eq('/tmp/scout_apm/scout_apm_log_monitor.pid')
     expect(conf.value('logs_ingest_key')).to eq('00001000010000abc')
-    expect(conf.value('logs_monitored')).to eq(['/tmp/fake_log_file.log'])
-  end
-
-  it 'loads the state file into the config' do
-    ENV['SCOUT_MONITOR_STATE_FILE'] = File.expand_path('../data/state_file.json', __dir__)
-
-    context = ScoutApm::Logging::Context.new
-    conf_file = File.expand_path('../data/config_test_1.yml', __dir__)
-    conf = ScoutApm::Logging::Config.with_file(context, conf_file)
-
-    expect(conf.value('health_check_port')).to eq(1234)
   end
 end

data/spec/unit/loggers/capture_spec.rb

@@ -16,8 +16,6 @@ end
 
 describe ScoutApm::Logging::Loggers::Capture do
   it 'should swap the STDOUT logger and create a proxy logger' do
-    ENV['SCOUT_MONITOR_INTERVAL'] = '10'
-    ENV['SCOUT_MONITOR_INTERVAL_DELAY'] = '10'
     ENV['SCOUT_LOGS_MONITOR'] = 'true'
     ENV['SCOUT_LOGS_CAPTURE_LEVEL'] = 'debug'
 
@@ -47,10 +45,6 @@ describe ScoutApm::Logging::Loggers::Capture do
       # Shouldn't capture. While the log_capture_level was set to debug,
       # the original logger instance had a higher log level of info.
       expect(content).not_to include('SHOULD NOT CAPTURE')
-
-      state_file = File.read(context.config.value('monitor_state_file'))
-      state_data = JSON.parse(state_file)
-      expect(state_data['logs_monitored']).to eq([log_path])
     end
 
     expect(output_from_log).to include('TEST')