scout_apm_logging 0.0.13 → 1.0.1

metadata

@@ -1,15 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm_logging
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 1.0.1
 platform: ruby
 authors:
 - Scout APM
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-06 00:00:00.000000000 Z
+date: 2024-10-09 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: googleapis-common-protos-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: google-protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: opentelemetry-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: opentelemetry-common
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: opentelemetry-instrumentation-base
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: opentelemetry-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: scout_apm
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +150,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.30.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Sets up log monitoring for Scout APM Ruby clients.
 email:
 - support@scoutapp.com
@@ -79,9 +177,9 @@ files:
 - CHANGELOG.md
 - Dockerfile
 - Gemfile
+- NOTICE
 - README.md
 - Rakefile
-- bin/scout_apm_logging_monitor
 - gems/rails.gemfile
 - lib/scout_apm/logging/config.rb
 - lib/scout_apm/logging/context.rb
@@ -89,46 +187,48 @@ files:
 - lib/scout_apm/logging/loggers/capture.rb
 - lib/scout_apm/logging/loggers/formatter.rb
 - lib/scout_apm/logging/loggers/logger.rb
+- lib/scout_apm/logging/loggers/opentelemetry/LICENSE
+- lib/scout_apm/logging/loggers/opentelemetry/NOTICE
+- lib/scout_apm/logging/loggers/opentelemetry/api/logs.rb
+- lib/scout_apm/logging/loggers/opentelemetry/api/logs/log_record.rb
+- lib/scout_apm/logging/loggers/opentelemetry/api/logs/logger.rb
+- lib/scout_apm/logging/loggers/opentelemetry/api/logs/logger_provider.rb
+- lib/scout_apm/logging/loggers/opentelemetry/api/logs/severity_number.rb
+- lib/scout_apm/logging/loggers/opentelemetry/api/logs/version.rb
+- lib/scout_apm/logging/loggers/opentelemetry/exporter/exporter/otlp/logs_exporter.rb
+- lib/scout_apm/logging/loggers/opentelemetry/exporter/exporter/otlp/version.rb
+- lib/scout_apm/logging/loggers/opentelemetry/exporter/proto/collector/logs/v1/logs_service_pb.rb
+- lib/scout_apm/logging/loggers/opentelemetry/exporter/proto/common/v1/common_pb.rb
+- lib/scout_apm/logging/loggers/opentelemetry/exporter/proto/logs/v1/logs_pb.rb
+- lib/scout_apm/logging/loggers/opentelemetry/exporter/proto/resource/v1/resource_pb.rb
+- lib/scout_apm/logging/loggers/opentelemetry/opentelemetry.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/export.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/export/batch_log_record_processor.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/export/log_record_exporter.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/log_record.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/log_record_data.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/log_record_processor.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/logger.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/logger_provider.rb
+- lib/scout_apm/logging/loggers/opentelemetry/sdk/logs/version.rb
 - lib/scout_apm/logging/loggers/patches/rails_logger.rb
 - lib/scout_apm/logging/loggers/patches/tagged_logging.rb
 - lib/scout_apm/logging/loggers/proxy.rb
 - lib/scout_apm/logging/loggers/swaps/rails.rb
 - lib/scout_apm/logging/loggers/swaps/scout.rb
 - lib/scout_apm/logging/loggers/swaps/sidekiq.rb
-- lib/scout_apm/logging/monitor/_rails.rb
-- lib/scout_apm/logging/monitor/collector/checksum.rb
-- lib/scout_apm/logging/monitor/collector/configuration.rb
-- lib/scout_apm/logging/monitor/collector/downloader.rb
-- lib/scout_apm/logging/monitor/collector/extractor.rb
-- lib/scout_apm/logging/monitor/collector/manager.rb
-- lib/scout_apm/logging/monitor/monitor.rb
-- lib/scout_apm/logging/monitor_manager/manager.rb
-- lib/scout_apm/logging/state.rb
 - lib/scout_apm/logging/utils.rb
 - lib/scout_apm/logging/version.rb
 - lib/scout_apm_logging.rb
 - scout_apm_logging.gemspec
 - spec/data/config_test_1.yml
-- spec/data/empty_logs_config.yml
-- spec/data/logs_config.yml
 - spec/data/mock_config.yml
-- spec/data/state_file.json
-- spec/integration/loggers/capture_spec.rb
-- spec/integration/monitor/collector/downloader/will_verify_checksum.rb
-- spec/integration/monitor/collector_healthcheck_spec.rb
-- spec/integration/monitor/continuous_state_collector_spec.rb
-- spec/integration/monitor/previous_collector_setup_spec.rb
-- spec/integration/monitor_manager/disable_agent_spec.rb
-- spec/integration/monitor_manager/monitor_pid_file_spec.rb
-- spec/integration/monitor_manager/single_monitor_spec.rb
 - spec/integration/rails/lifecycle_spec.rb
 - spec/rails/app.rb
 - spec/spec_helper.rb
 - spec/unit/config_spec.rb
 - spec/unit/loggers/capture_spec.rb
-- spec/unit/monitor/collector/configuration_spec.rb
-- spec/unit/state_spec.rb
-- tooling/checksums.rb
 homepage: https://github.com/scoutapp/scout_apm_ruby_logging
 licenses:
 - MIT
@@ -154,22 +254,9 @@ specification_version: 4
 summary: Ruby Logging Support
 test_files:
 - spec/data/config_test_1.yml
-- spec/data/empty_logs_config.yml
-- spec/data/logs_config.yml
 - spec/data/mock_config.yml
-- spec/data/state_file.json
-- spec/integration/loggers/capture_spec.rb
-- spec/integration/monitor/collector/downloader/will_verify_checksum.rb
-- spec/integration/monitor/collector_healthcheck_spec.rb
-- spec/integration/monitor/continuous_state_collector_spec.rb
-- spec/integration/monitor/previous_collector_setup_spec.rb
-- spec/integration/monitor_manager/disable_agent_spec.rb
-- spec/integration/monitor_manager/monitor_pid_file_spec.rb
-- spec/integration/monitor_manager/single_monitor_spec.rb
 - spec/integration/rails/lifecycle_spec.rb
 - spec/rails/app.rb
 - spec/spec_helper.rb
 - spec/unit/config_spec.rb
 - spec/unit/loggers/capture_spec.rb
-- spec/unit/monitor/collector/configuration_spec.rb
-- spec/unit/state_spec.rb

data/bin/scout_apm_logging_monitor

@@ -1,6 +0,0 @@
-#!/usr/bin/env ruby
-
-require_relative '../lib/scout_apm/logging/monitor/_rails.rb'
-require_relative "../lib/scout_apm/logging/monitor/monitor.rb"
-
-ScoutApm::Logging::Monitor.instance.setup!

data/lib/scout_apm/logging/monitor/_rails.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-# We start the monitor process outside the context of Rails, and ultimately don't use
-# it for anything related to starting the collector. However, when we load the config/scout_apm.yml file,
-# we support ERB, and users may be using Rails.env methods for naming the app, or configuring whether
-# monitor should be enabled for a specific environment.
-
-require 'active_support'
-
-# https://github.com/rails/rails/blob/v7.2.1/railties/lib/rails.rb#L76
-module Rails
-  class << self
-    def env
-      # EnvironmentInquirer was added in Rails 6.1
-      @env ||= if const_defined?('::ActiveSupport::EnvironmentInquirer')
-                 ::ActiveSupport::EnvironmentInquirer.new(ENV['RAILS_ENV'].presence || ENV['RACK_ENV'].presence || 'development')
-               else
-                 ::ActiveSupport::StringInquirer.new(ENV['RAILS_ENV'].presence || ENV['RACK_ENV'].presence || 'development')
-               end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor/collector/checksum.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    module Collector
-      # Contains logic around verifying the checksum of the otelcol-contrib binary.
-      class Checksum
-        attr_reader :context
-
-        KNOWN_CHECKSUMS = {
-          'darwin_amd64' => '5456734e124221e7ff775c52bd3693d05b3fac43ebe06b22aa5f220f1962ed8c',
-          'darwin_arm64' => 'f9564560798ac5c099885903f303fcda97b7ea649ec299e075b72f3805873879',
-          'linux_amd64' => '326772622016f7ff7e966a7ae8a0f439dc49a3d80b6d79a82b62608af447e851',
-          'linux_arm64' => '73d797817540363a37f27e32270f98053ed17b1df36df2d30db1715ce40f4cff'
-        }.freeze
-
-        def initialize(context)
-          @context = context
-        end
-
-        def verified_checksum?(should_log_failures: false)
-          return false unless File.exist?(collector_tar_path)
-
-          checksum = `sha256sum #{collector_tar_path}`.split(' ').first
-          same_checksum_result = checksum == KNOWN_CHECKSUMS[double]
-
-          log_failed_checksum if !same_checksum_result && should_log_failures
-          same_checksum_result
-        end
-
-        def log_failed_checksum
-          if KNOWN_CHECKSUMS.key?(double)
-            context.logger.error('Checksum verification failed for otelcol.tar.gz.')
-          else
-            context.logger.error("Checksum verification failed for otelcol.tar.gz. Unknown architecture: #{double}")
-          end
-        end
-
-        private
-
-        def double
-          "#{Utils.get_host_os}_#{Utils.get_architecture}"
-        end
-
-        def collector_tar_path
-          "#{context.config.value('collector_download_dir')}/otelcol.tar.gz"
-        end
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor/collector/configuration.rb

@@ -1,150 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    module Collector
-      # Adds a method to Hash similar to that of the Rails deep_merge.
-      module HashDeepMerge
-        refine Hash do
-          def deep_merge(second)
-            merger = proc { |_, v1, v2|
-              if v1.is_a?(Hash) && v2.is_a?(Hash)
-                v1.merge(v2, &merger)
-              elsif v1.is_a?(Array) && v2.is_a?(Array)
-                v1 | v2
-              else
-                [:undefined, nil, :nil].include?(v2) ? v1 : v2
-              end
-            }
-            merge(second.to_h, &merger)
-          end
-        end
-      end
-
-      # Creates the configuration to be used when launching the collector.
-      class Configuration
-        using HashDeepMerge
-
-        attr_reader :context
-
-        def initialize(context)
-          @context = context
-        end
-
-        def setup!
-          create_storage_directories
-
-          create_config_file
-        end
-
-        def create_config_file
-          contents = YAML.dump(combined_contents)
-          File.write(config_file, contents)
-        end
-
-        private
-
-        def create_storage_directories
-          # Sending queue storage directory
-          Utils.ensure_directory_exists(context.config.value('collector_sending_queue_storage_dir'))
-          # Offset storage directory
-          Utils.ensure_directory_exists(context.config.value('collector_offset_storage_dir'))
-        end
-
-        def combined_contents
-          default_contents = YAML.safe_load(config_contents)
-
-          default_contents.deep_merge(loaded_config_contents)
-        end
-
-        def loaded_config_contents
-          config_path = context.config.value('logs_config')
-
-          if config_path && File.exist?(config_path)
-            YAML.load_file(config_path) || {}
-          elsif File.exist?(assumed_config_file_path)
-            YAML.load_file(assumed_config_file_path) || {}
-          else
-            {}
-          end
-        end
-
-        def config_file
-          context.config.value('collector_config_file')
-        end
-
-        def config_contents # rubocop:disable Metrics/AbcSize
-          <<~CONFIG
-            receivers:
-              filelog:
-                include: [#{context.config.value('logs_monitored').join(',')}]
-                storage: file_storage/filelogreceiver
-                operators:
-                  - type: json_parser
-                    severity:
-                      parse_from: attributes.severity
-                    timestamp:
-                      parse_from: attributes.time
-                      layout: "%Y-%m-%dT%H:%M:%S.%LZ"
-            processors:
-              transform:
-                log_statements:
-                  - context: log
-                    statements:
-                    # Copy original body to raw_bytes attribute.
-                    - 'set(attributes["raw_bytes"], body)'
-                    # Replace the body with the log message.
-                    - 'set(body, attributes["msg"])'
-                    # Remove the msg attribute.
-                    - 'delete_key(attributes, "msg")'
-                    # Move service.name attribute to resource attribute.
-                    - 'set(resource.attributes["service.name"], attributes["service.name"])'
-              batch:
-            exporters:
-              otlp:
-                endpoint: #{context.config.value('logs_reporting_endpoint')}
-                headers:
-                  x-telemetryhub-key: #{context.config.value('logs_ingest_key')}
-                sending_queue:
-                  storage: file_storage/otc
-            extensions:
-              health_check:
-                endpoint: #{health_check_endpoint}
-              file_storage/filelogreceiver:
-                directory: #{context.config.value('collector_offset_storage_dir')}
-              file_storage/otc:
-                directory: #{context.config.value('collector_sending_queue_storage_dir')}
-                timeout: 10s
-            service:
-              extensions:
-                - health_check
-                - file_storage/filelogreceiver
-                - file_storage/otc
-              pipelines:
-                logs:
-                  receivers:
-                    - filelog
-                  processors:
-                    - transform
-                    - batch
-                  exporters:
-                    - otlp
-              telemetry:
-                metrics:
-                  level: none
-                logs:
-                  level: #{context.config.value('collector_log_level')}
-          CONFIG
-        end
-
-        def health_check_endpoint
-          "localhost:#{context.config.value('health_check_port')}"
-        end
-
-        def assumed_config_file_path
-          "#{context.application_root}/config/scout_logs_config.yml"
-        end
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor/collector/downloader.rb

@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    module Collector
-      # Downloads the collector-contrib binary from the OpenTelemetry project.
-      class Downloader
-        attr_accessor :failed_count
-        attr_reader :context, :checksum
-
-        def initialize(context)
-          @context = context
-          @checksum = Checksum.new(context)
-        end
-
-        def download!
-          # Already downloaded the collector. Noop.
-          return if checksum.verified_checksum?
-
-          # Account for issues such as failed extractions or download corruptions.
-          download_collector
-          verify_checksum
-        rescue StandardError => e
-          # Bypass Rubcop useless asignment rule.
-          failed_count ||= 0
-
-          if failed_count < 3
-            context.logger.error("Failed to download or extract otelcol-contrib: #{e}. Retrying...")
-            failed_count += 1
-            retry
-          end
-        end
-
-        def download_collector(url = nil, redirect: false) # rubocop:disable Metrics/AbcSize
-          # Prevent double logging.
-          unless redirect
-            context.logger.debug("Downloading otelcol-contrib for version #{context.config.value('collector_version')}")
-          end
-
-          url_to_download = url || collector_url
-          uri = URI(url_to_download)
-
-          Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
-            request = Net::HTTP::Get.new(uri)
-            http.request(request) do |response|
-              return download_collector(response['location'], redirect: true) if response.code == '302'
-
-              File.open(destination, 'wb') do |file|
-                response.read_body do |chunk|
-                  file.write(chunk)
-                end
-              end
-            end
-          end
-        end
-
-        private
-
-        def verify_checksum
-          raise 'Invalid checksum on download.' unless checksum.verified_checksum?
-        end
-
-        def collector_url
-          collector_version = context.config.value('collector_version')
-          architecture = Utils.get_architecture
-          host_os = Utils.get_host_os
-
-          # https://opentelemetry.io/docs/collector/installation/#manual-linux-installation
-          "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v#{collector_version}/otelcol-contrib_#{collector_version}_#{host_os}_#{architecture}.tar.gz"
-        end
-
-        def destination
-          "#{context.config.value('collector_download_dir')}/otelcol.tar.gz"
-        end
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor/collector/extractor.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module ScoutApm
-  module Logging
-    module Collector
-      # Extracts the contents of the collector tar file.
-      class Extractor
-        attr_reader :context
-
-        def initialize(context)
-          @context = context
-        end
-
-        def extract!
-          # Already extracted. Noop.
-          return if has_been_extracted?
-
-          system("tar -xzf #{tar_path} -C #{context.config.value('collector_download_dir')}")
-        end
-
-        def has_been_extracted?
-          File.exist?(binary_path)
-        end
-
-        private
-
-        def tar_path
-          "#{context.config.value('collector_download_dir')}/otelcol.tar.gz"
-        end
-
-        def binary_path
-          "#{context.config.value('collector_download_dir')}/otelcol-contrib"
-        end
-      end
-    end
-  end
-end

data/lib/scout_apm/logging/monitor/collector/manager.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-require_relative './checksum'
-require_relative './configuration'
-require_relative './downloader'
-require_relative './extractor'
-
-module ScoutApm
-  module Logging
-    module Collector
-      # Manager class for the downloading, configuring, and starting of the collector.
-      class Manager
-        attr_reader :context
-
-        def initialize(context)
-          @context = context
-
-          @checksum = Checksum.new(@context)
-          @configuration = Configuration.new(@context)
-          @downloader = Downloader.new(@context)
-          @extractor = Extractor.new(@context)
-        end
-
-        def setup!
-          @configuration.setup!
-          @downloader.download!
-          @extractor.extract!
-
-          start_collector if verified_checksum_and_extracted?
-        end
-
-        def start_collector
-          context.logger.info('Starting otelcol-contrib')
-          collector_process = Process.spawn("#{extracted_collector_path}/otelcol-contrib --config #{config_file}")
-          File.write(context.config.value('collector_pid_file'), collector_process)
-        end
-
-        private
-
-        def verified_checksum_and_extracted?
-          has_verfied_checksum = @checksum.verified_checksum?(should_log_failures: true)
-          has_extracted_content = @extractor.has_been_extracted?
-
-          has_verfied_checksum && has_extracted_content
-        end
-
-        def extracted_collector_path
-          context.config.value('collector_download_dir')
-        end
-
-        def config_file
-          context.config.value('collector_config_file')
-        end
-      end
-    end
-  end
-end