semantic_logger 4.16.1 → 4.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 475a3c63ee87f78516ad0e695879d79af03c5d68a86c129f0c128f0faaa8ad82
-  data.tar.gz: cebcd2653fc6d08090f63070a52147e5e1878c34407c8b1fd987b3959f4b5955
+  metadata.gz: a10d5e24ca0c3542ec98c53268bd56f96c3432fd7b8de39c55d0f7938f5b6d44
+  data.tar.gz: 95bdc9df3003bf78c4146847303e8d345600a587fb6fd817343f8822fc62f42b
 SHA512:
-  metadata.gz: 7d988c5776532787067b5f2b82cac36e41e51a2a93db68f5fc2fb8ee5f278d4d5b2462bd78819a166a725289930a7bc128a5753ed5415409a74666408471b052
-  data.tar.gz: 84ea17d324b85502ae1f48c3ffa220762035fb9eaf180057f35e355a11224ad3583524953b14ab8c47fe129a152cf78fe210702d04f75929d161afd4c9d72ecc
+  metadata.gz: 1167fd1b55ffa9805721f598c5ee3e53f9759562ec86088c1237d6c21fc2359c58edf4e6dd81ac91016a36f470636f70ac120939b255e62c0a075a3e6a2190d4
+  data.tar.gz: 815b6f851d8f7c9ccee2f3e8185c66176bd9263becdab417b8afef70b9aa3e005e78864b135f5c06d6f3a201da3070fe1fe046aee68922c15f9be0170ae9af07

data/README.md

@@ -27,6 +27,7 @@ Logging to the following destinations are all supported "out-of-the-box":
 * TCP
 * UDP
 * Syslog
+* CloudWatch Logs
 * Add any existing Ruby logger as another destination.
 * Roll-your-own
 

data/lib/semantic_logger/appender/async.rb

@@ -40,7 +40,6 @@ module SemanticLogger
                      max_queue_size: 10_000,
                      lag_check_interval: 1_000,
                      lag_threshold_s: 30)
-
         @appender           = appender
         @lag_check_interval = lag_check_interval
         @lag_threshold_s    = lag_threshold_s

data/lib/semantic_logger/appender/async_batch.rb

@@ -30,7 +30,6 @@ module SemanticLogger
                      lag_threshold_s: 30,
                      batch_size: 300,
                      batch_seconds: 5)
-
         @batch_size    = batch_size
         @batch_seconds = batch_seconds
         @signal        = Concurrent::Event.new
@@ -47,7 +46,7 @@ module SemanticLogger
 
       # Add log message for processing.
       def log(log)
-        result = super(log)
+        result = super
         # Wake up the processing thread since the number of queued messages has been exceeded.
         signal.set if queue.size >= batch_size
         result

data/lib/semantic_logger/appender/bugsnag.rb

@@ -30,12 +30,13 @@ module SemanticLogger
       #     Proc: Only include log messages where the supplied Proc returns true
       #           The Proc must return true or false.
       def initialize(level: :error, **args, &block)
-        raise "Bugsnag only supports :info, :warn, or :error log levels" unless %i[info warn error fatal].include?(level)
+        raise "Bugsnag only supports :info, :warn, or :error log levels" unless %i[info warn error
+                                                                                   fatal].include?(level)
 
         # Replace the Bugsnag logger so that we can identify its log messages and not forward them to Bugsnag
         ::Bugsnag.configure { |config| config.logger = SemanticLogger[Bugsnag] }
 
-        super(level: level, **args, &block)
+        super
       end
 
       # Returns [Hash] of parameters to send to Bugsnag.

data/lib/semantic_logger/appender/cloudwatch_logs.rb

@@ -0,0 +1,150 @@
+begin
+  require "aws-sdk-cloudwatchlogs"
+rescue LoadError
+  raise LoadError,
+        'Gem aws-sdk-cloudwatchlogs is required for logging to CloudWatch Logs. Please add the gem "aws-sdk-cloudwatchlogs" to your Gemfile.'
+end
+
+require "concurrent"
+
+# Forward all log messages to CloudWatch Logs.
+#
+# Example:
+#
+#   SemanticLogger.add_appender(
+#     appender: :cloudwatch_logs,
+#     client_kwargs: {region: "eu-west-1"},
+#     group: "/my/application",
+#     create_stream: true
+#   )
+module SemanticLogger
+  module Appender
+    class CloudwatchLogs < SemanticLogger::Subscriber
+      attr_reader :client_kwargs, :group, :create_group, :create_stream, :force_flush_interval_seconds, :max_buffered_events,
+                  :task, :client, :buffered_logs
+
+      # Create CloudWatch Logs Appender
+      #
+      # Parameters:
+      #   group: [String]
+      #     Log group name
+      #
+      #   client_kwargs: [Hash]
+      #     A hash to be passed to Aws::CloudWatchLogs::Client.new
+      #     Default: {}
+      #
+      #   stream: [String]
+      #     Log stream name
+      #     Default: SemanticLogger.host
+      #
+      #   create_group: [Boolean]
+      #     If the missing log group should be automatically created.
+      #     Default: false
+      #
+      #   create_stream: [Boolean]
+      #     If the missing log stream should be automatically created.
+      #     Default: true
+      #
+      #   force_flush_interval_seconds: [Integer]
+      #     Flush buffered logs every X seconds, regardless of the current buffer size.
+      #     Default: 5
+      #
+      #   max_buffered_events: [Integer]
+      #     Flush buffered logs if they are above the currently set size.
+      #     Note that currently CloudWatch Logs has 10000 hard limit.
+      #     Default: 4000
+      def initialize(
+        *args,
+        group:,
+        client_kwargs: {},
+        stream: nil,
+        create_group: false,
+        create_stream: true,
+        force_flush_interval_seconds: 5,
+        max_buffered_events: 4_000,
+        **kwargs,
+        &block
+      )
+        @group = group
+        @client_kwargs = client_kwargs
+        @stream = stream
+        @create_group = create_group
+        @create_stream = create_stream
+        @force_flush_interval_seconds = force_flush_interval_seconds
+        @max_buffered_events = max_buffered_events
+
+        super(*args, **kwargs, &block)
+        reopen
+      end
+
+      # Method called to log an event
+      def log(log)
+        buffered_logs << log
+
+        put_log_events if buffered_logs.size >= max_buffered_events
+      end
+
+      def flush
+        task.execute while buffered_logs.size.positive?
+      end
+
+      def close
+        task.shutdown
+      end
+
+      def reopen
+        @buffered_logs = Concurrent::Array.new
+        @client = Aws::CloudWatchLogs::Client.new(client_kwargs)
+
+        @task = Concurrent::TimerTask.new(execution_interval: force_flush_interval_seconds,
+                                          interval_type:      :fixed_rate) do
+          put_log_events
+        end
+        @task.execute
+      end
+
+      # Use JSON Formatter by default
+      def default_formatter
+        SemanticLogger::Formatters::Json.new
+      end
+
+      private
+
+      def put_log_events
+        logs = buffered_logs.shift(max_buffered_events)
+
+        return if logs.none?
+
+        begin
+          client.put_log_events({
+                                  log_group_name:  group,
+                                  log_stream_name: stream,
+                                  log_events:      logs.map do |log|
+                                                     {
+                                                       timestamp: (log.time.to_f * 1000).floor,
+                                                       message:   formatter.call(log, self)
+                                                     }
+                                                   end
+                                })
+        rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
+          if e.message.include?("log group does not exist.") && create_group
+            client.create_log_group({
+                                      log_group_name: group
+                                    })
+            retry
+          elsif e.message.include?("log stream does not exist.") && create_stream
+            client.create_log_stream({
+                                       log_group_name:  group,
+                                       log_stream_name: stream
+                                     })
+            retry
+          end
+        end
+      end
+
+      def stream
+        @stream || host
+      end
+    end
+  end
+end

data/lib/semantic_logger/appender/elasticsearch.rb

@@ -139,7 +139,6 @@ module SemanticLogger
                      data_stream: false,
                      **elasticsearch_args,
                      &block)
-
         @url                         = url
         @index                       = index
         @date_pattern                = date_pattern

data/lib/semantic_logger/appender/elasticsearch_http.rb

@@ -56,7 +56,6 @@ module SemanticLogger
                      url: "http://localhost:9200",
                      **http_args,
                      &block)
-
         @index = index
         @type  = type
         super(url: url, **http_args, &block)

data/lib/semantic_logger/appender/file.rb

@@ -122,7 +122,8 @@ module SemanticLogger
       #
       #    logger = SemanticLogger["test"]
       #    logger.info "Hello World"
-      def initialize(file_name, retry_count: 1, append: true, reopen_period: nil, reopen_count: 0, reopen_size: 0, encoding: Encoding::BINARY, exclusive_lock: false, **args, &block)
+      def initialize(file_name, retry_count: 1, append: true, reopen_period: nil, reopen_count: 0, reopen_size: 0,
+                     encoding: Encoding::BINARY, exclusive_lock: false, **args, &block)
         if !file_name.is_a?(String) || file_name.empty?
           raise(ArgumentError, "SemanticLogging::Appender::File file_name must be a non-empty string")
         end
@@ -267,7 +268,10 @@ module SemanticLogger
 
         duration = match[1]
         period   = match[2]
-        raise(ArgumentError, "Invalid or missing duration in: #{period_string}, must begin with an integer.") unless duration
+        unless duration
+          raise(ArgumentError,
+                "Invalid or missing duration in: #{period_string}, must begin with an integer.")
+        end
         raise(ArgumentError, "Invalid or missing period in: #{period_string}, must end with m,h, or d.") unless period
 
         [duration.to_i, period]

data/lib/semantic_logger/appender/graylog.rb

@@ -25,7 +25,8 @@ module SemanticLogger
       class LevelMap
         attr_accessor :trace, :debug, :info, :warn, :error, :fatal
 
-        def initialize(trace: GELF::DEBUG, debug: GELF::DEBUG, info: GELF::INFO, warn: GELF::WARN, error: GELF::ERROR, fatal: GELF::FATAL)
+        def initialize(trace: GELF::DEBUG, debug: GELF::DEBUG, info: GELF::INFO, warn: GELF::WARN, error: GELF::ERROR,
+                       fatal: GELF::FATAL)
           @trace = trace
           @debug = debug
           @info  = info
@@ -88,7 +89,6 @@ module SemanticLogger
                      level_map: LevelMap.new,
                      **args,
                      &block)
-
         @url          = url
         @max_size     = max_size
         @gelf_options = gelf_options

data/lib/semantic_logger/appender/honeybadger.rb

@@ -38,7 +38,7 @@ module SemanticLogger
       #     Name of this application to appear in log messages.
       #     Default: SemanticLogger.application
       def initialize(level: :error, **args, &block)
-        super(level: level, **args, &block)
+        super
       end
 
       # Send an error notification to honeybadger

data/lib/semantic_logger/appender/honeybadger_insights.rb

@@ -38,7 +38,7 @@ module SemanticLogger
       #     Name of this application to appear in log messages.
       #     Default: SemanticLogger.application
       def initialize(level: :info, **args, &block)
-        super(level: level, **args, &block)
+        super
       end
 
       # Send log to honeybadger events API

data/lib/semantic_logger/appender/http.rb

@@ -107,7 +107,6 @@ module SemanticLogger
                      continue_timeout: 1.0,
                      **args,
                      &block)
-
         @url              = url
         @proxy_url        = proxy_url
         @ssl_options      = ssl

data/lib/semantic_logger/appender/kafka.rb

@@ -1,7 +1,8 @@
 begin
   require "kafka"
 rescue LoadError
-  raise LoadError, 'Gem ruby-kafka is required for logging to Elasticsearch. Please add the gem "ruby-kafka" to your Gemfile.'
+  raise LoadError,
+        'Gem ruby-kafka is required for logging to Elasticsearch. Please add the gem "ruby-kafka" to your Gemfile.'
 end
 
 require "date"
@@ -127,7 +128,6 @@ module SemanticLogger
                      topic: "log_messages", partition: nil, partition_key: nil, key: nil,
                      delivery_threshold: 100, delivery_interval: 10, required_acks: 1,
                      metrics: true, **args, &block)
-
         @seed_brokers             = seed_brokers
         @client_id                = client_id
         @connect_timeout          = connect_timeout

data/lib/semantic_logger/appender/loki.rb

@@ -0,0 +1,62 @@
+# Forward application metrics to a Loki instance using HTTP push API
+#
+# Example:
+#   SemanticLogger.add_appender(
+#     appender: :loki,
+#     url: "https://logs-prod-001.grafana.net",
+#     username: "grafana_username",
+#     password: "grafana_token_here",
+#     compress: true
+#   )
+module SemanticLogger
+  module Appender
+    class Loki < SemanticLogger::Appender::Http
+      INGESTION_PATH = "loki/api/v1/push".freeze
+
+      # Create Grafana Loki appender.
+      #
+      # Parameters:
+      #   filter: [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied
+      #             regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true.
+      #           The Proc must return true or false.
+      #
+      #   host: [String]
+      #     Name of this host to send as a dimension.
+      #     Default: SemanticLogger.host
+      #
+      #   application: [String]
+      #     Name of this application to send as a dimension.
+      #     Default: SemanticLogger.application
+      #
+      #   url: [String]
+      #     Define the loki instance URL.
+      #     Example: https://logs-prod-999.grafana.net
+      #     Default: nil
+      def initialize(url: nil,
+                     formatter: SemanticLogger::Formatters::Loki.new,
+                     header: {"Content-Type" => "application/json"},
+                     path: INGESTION_PATH,
+                     **args,
+                     &block)
+
+        super(url: "#{url}/#{path}", formatter: formatter, header: header, **args, &block)
+      end
+
+      def log(log)
+        message = formatter.call(log, self)
+        puts message
+        logger.trace(message)
+        post(message)
+      end
+
+      # Logs in batches
+      def batch(logs)
+        message = formatter.batch(logs, self)
+        logger.trace(message)
+        post(message)
+      end
+    end
+  end
+end

data/lib/semantic_logger/appender/mongodb.rb

@@ -2,7 +2,8 @@ require "socket"
 begin
   require "mongo"
 rescue LoadError
-  raise LoadError, 'Gem mongo is required for logging to MongoDB. Please add the gem "mongo" v2.0 or greater to your Gemfile.'
+  raise LoadError,
+        'Gem mongo is required for logging to MongoDB. Please add the gem "mongo" v2.0 or greater to your Gemfile.'
 end
 
 module SemanticLogger
@@ -110,7 +111,6 @@ module SemanticLogger
                      collection_max: nil,
                      **args,
                      &block)
-
         @client          = Mongo::Client.new(uri, logger: logger)
         @collection_name = collection_name
         @options         = {

data/lib/semantic_logger/appender/new_relic.rb

@@ -1,7 +1,8 @@
 begin
   require "newrelic_rpm"
 rescue LoadError
-  raise LoadError, 'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
+  raise LoadError,
+        'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
 end
 
 # Send log messages to NewRelic
@@ -32,7 +33,7 @@ module SemanticLogger
       #     Proc: Only include log messages where the supplied Proc returns true
       #           The Proc must return true or false.
       def initialize(level: :error, **args, &block)
-        super(level: level, **args, &block)
+        super
       end
 
       # Returns [Hash] of parameters to send to New Relic.

data/lib/semantic_logger/appender/new_relic_logs.rb

@@ -1,7 +1,8 @@
 begin
   require "newrelic_rpm"
 rescue LoadError
-  raise LoadError, 'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
+  raise LoadError,
+        'Gem newrelic_rpm is required for logging to New Relic. Please add the gem "newrelic_rpm" to your Gemfile.'
 end
 
 require "semantic_logger/formatters/new_relic_logs"
@@ -40,17 +41,27 @@ module SemanticLogger
       #     Proc: Only include log messages where the supplied Proc returns true
       #           The Proc must return true or false.
       def initialize(formatter: SemanticLogger::Formatters::NewRelicLogs.new, **args, &block)
-        super(formatter: formatter, **args, &block)
+        super
       end
 
       # Send an error notification to New Relic
       def log(log)
-        self.class.log_newrelic(formatter.call(log, self).to_json, log.level.to_s.upcase)
+        begin
+          message = formatter.call(log, self) # Generate the structured log
+          json_message = message.to_json      # Convert the log to JSON
+          level = log.level.to_s.upcase       # Determine the log level
+          self.class.log_newrelic(json_message, level)
+        rescue JSON::GeneratorError => e
+          warn("Failed to serialize log message to JSON: #{e.message}")
+          warn("Problematic data: #{message.inspect}")
+        rescue StandardError => e
+          warn("Unexpected error while logging to New Relic: #{e.message}")
+        end
         true
       end
 
-      def self.log_newrelic(message, level)
-        ::NewRelic::Agent.agent.log_event_aggregator.record(message, level)
+      def self.log_newrelic(json_message, level)
+        ::NewRelic::Agent.agent.log_event_aggregator.record(json_message, level)
       end
     end
   end

data/lib/semantic_logger/appender/open_telemetry.rb

@@ -0,0 +1,83 @@
+begin
+  require "opentelemetry/logs"
+rescue LoadError
+  raise LoadError,
+        'Gem opentelemetry-logs-sdk is required for logging to Open Telemetry. Please add the gem "opentelemetry-logs-sdk" to your Gemfile.'
+end
+
+# Open Telemetry Appender
+#
+#   Writes log messages, and metrics to Open Telemetry.
+#
+module SemanticLogger
+  module Appender
+    class OpenTelemetry < SemanticLogger::Subscriber
+      attr_reader :name, :version, :logger
+
+      CAPTURE_CONTEXT = ->(log) { log.set_context(:open_telemetry, ::OpenTelemetry::Context.current) }
+
+      # Create a Open Telemetry Logger appender instance.
+      #
+      # Metric only log events are sent to the Open Telemetry Metrics API instead of the Logs API.
+      #   I.e. A metric without a message or an exception.
+      #   To disable this default behavior set `metrics: false`
+      #
+      # Example
+      #    SemanticLogger.add_appender(appender: :open_telemetry)
+      def initialize(name: "SemanticLogger",
+                     version: SemanticLogger::VERSION,
+                     formatter: SemanticLogger::Formatters::OpenTelemetry.new,
+                     metrics: true,
+                     **args,
+                     &block)
+        @name    = name
+        @version = version
+        @logger  = ::OpenTelemetry.logger_provider.logger(name: @name, version: @version)
+
+        # Capture the current Open Telemetry context when a log entry is captured.
+        # Prevents duplicate subscribers as long as it is from a constant.
+        SemanticLogger.on_log(CAPTURE_CONTEXT)
+
+        super(formatter: formatter, metrics: metrics, **args, &block)
+      end
+
+      def log(log)
+        # return log_metric(log) if metrics && log.metric_only?
+
+        body        = formatter.call(log, self)
+        level       = body.delete(:level)
+        level_index = body.delete(:level_index)
+        time        = body.delete(:time)
+        payload     = body.delete(:payload)
+
+        @logger.on_emit(
+          severity_text:   level,
+          severity_number: level_index,
+          timestamp:       time,
+          body:            body.transform_keys!(&:to_s),
+          attributes:      payload,
+          context:         log.context[:open_telemetry] || ::OpenTelemetry::Context.current
+        )
+        true
+      end
+
+      # Flush all pending logs.
+      def flush
+        @logger.logger_provider.force_flush
+      end
+
+      # Flush pending logs and close the appender
+      def close
+        @logger.logger_provider.shutdown
+      end
+
+      # For logging metrics only log events.
+      # def log_metric(log)
+      #   puts "**** TODO: Metric Only Event ****"
+      #   ap formatter.call(log, self)
+      #   ap log.payload
+      #   true
+      # end
+    end
+  end
+end

data/lib/semantic_logger/appender/sentry.rb

@@ -1,7 +1,8 @@
 begin
   require "sentry-raven"
 rescue LoadError
-  raise LoadError, 'Gem sentry-raven is required for logging purposes. Please add the gem "sentry-raven" to your Gemfile.'
+  raise LoadError,
+        'Gem sentry-raven is required for logging purposes. Please add the gem "sentry-raven" to your Gemfile.'
 end
 
 # Send log messages to sentry
@@ -40,7 +41,7 @@ module SemanticLogger
       def initialize(level: :error, **args, &block)
         # Replace the Sentry Raven logger so that we can identify its log messages and not forward them to Sentry
         Raven.configure { |config| config.logger = SemanticLogger[Raven] }
-        super(level: level, **args, &block)
+        super
       end
 
       # Send an error notification to sentry

data/lib/semantic_logger/appender/sentry_ruby.rb

@@ -41,7 +41,7 @@ module SemanticLogger
         # Replace the Sentry Ruby logger so that we can identify its log
         # messages and not forward them to Sentry
         ::Sentry.init { |config| config.logger = SemanticLogger[::Sentry] } unless ::Sentry.initialized?
-        super(level: level, **args, &block)
+        super
       end
 
       # Send an error notification to sentry

data/lib/semantic_logger/appender/splunk_http.rb

@@ -74,7 +74,6 @@ module SemanticLogger
                      compress: true,
                      **args,
                      &block)
-
         @source_type = source_type
         @index       = index
 

data/lib/semantic_logger/appender/syslog.rb

@@ -131,7 +131,6 @@ module SemanticLogger
                      tcp_client: {},
                      **args,
                      &block)
-
         @options            = options
         @facility           = facility
         @max_size           = max_size
@@ -215,7 +214,7 @@ module SemanticLogger
       def default_formatter
         if protocol == :syslog
           # Format is text output without the time
-          SemanticLogger::Formatters::Default.new(time_format: nil)
+          SemanticLogger::Formatters::Default.new(time_format: :notime)
         else
           SemanticLogger::Formatters::Syslog.new(facility: facility, level_map: level_map, max_size: max_size)
         end

data/lib/semantic_logger/appender/tcp.rb

@@ -1,7 +1,8 @@
 begin
   require "net/tcp_client"
 rescue LoadError
-  raise LoadError, 'Gem net_tcp_client is required for logging over TCP. Please add the gem "net_tcp_client" to your Gemfile.'
+  raise LoadError,
+        'Gem net_tcp_client is required for logging over TCP. Please add the gem "net_tcp_client" to your Gemfile.'
 end
 
 raise "Net::TCPClient v2.0 or greater is required to log over TCP" unless Net::TCPClient::VERSION.to_f >= 2.0

data/lib/semantic_logger/appender.rb

@@ -4,6 +4,7 @@ module SemanticLogger
     autoload :Async,               "semantic_logger/appender/async"
     autoload :AsyncBatch,          "semantic_logger/appender/async_batch"
     autoload :Bugsnag,             "semantic_logger/appender/bugsnag"
+    autoload :CloudwatchLogs,      "semantic_logger/appender/cloudwatch_logs"
     autoload :Elasticsearch,       "semantic_logger/appender/elasticsearch"
     autoload :ElasticsearchHttp,   "semantic_logger/appender/elasticsearch_http"
     autoload :File,                "semantic_logger/appender/file"
@@ -17,6 +18,7 @@ module SemanticLogger
     autoload :MongoDB,             "semantic_logger/appender/mongodb"
     autoload :NewRelic,            "semantic_logger/appender/new_relic"
     autoload :NewRelicLogs,        "semantic_logger/appender/new_relic_logs"
+    autoload :OpenTelemetry,       "semantic_logger/appender/open_telemetry"
     autoload :Rabbitmq,            "semantic_logger/appender/rabbitmq"
     autoload :Splunk,              "semantic_logger/appender/splunk"
     autoload :SplunkHttp,          "semantic_logger/appender/splunk_http"
@@ -25,6 +27,7 @@ module SemanticLogger
     autoload :Udp,                 "semantic_logger/appender/udp"
     autoload :Wrapper,             "semantic_logger/appender/wrapper"
     autoload :SentryRuby,          "semantic_logger/appender/sentry_ruby"
+    autoload :Loki,                "semantic_logger/appender/loki"
     # @formatter:on
 
     # Returns [SemanticLogger::Subscriber] appender for the supplied options