semantic_logger 4.18.0 → 5.0.0

data/lib/semantic_logger/appender/elasticsearch.rb

@@ -5,7 +5,7 @@ rescue LoadError
         'Gem elasticsearch is required for logging to Elasticsearch. Please add the gem "elasticsearch" to your Gemfile.'
 end
 
-require "date"
+require "semantic_logger/appender/elasticsearch_base"
 
 # Forward all log messages to Elasticsearch.
 #
@@ -17,190 +17,14 @@ require "date"
 #   )
 module SemanticLogger
   module Appender
-    class Elasticsearch < SemanticLogger::Subscriber
-      attr_accessor :url, :index, :date_pattern, :type, :client, :flush_interval, :timeout_interval, :batch_size,
-                    :elasticsearch_args
-
-      # Create Elasticsearch appender over persistent HTTP(S)
-      #
-      # Parameters:
-      #   index: [String]
-      #     Prefix of the index to store the logs in Elasticsearch.
-      #     The final index appends the date so that indexes are used per day.
-      #       I.e. The final index will look like 'semantic_logger-YYYY.MM.DD'
-      #     Default: 'semantic_logger'
-      #
-      #   date_pattern: [String]
-      #     The time format used to generate the full index name. Useful
-      #       if you want monthly indexes ('%Y.%m') or weekly ('%Y.%W').
-      #     Default: '%Y.%m.%d'
-      #
-      #   type: [String]
-      #     Document type to associate with logs when they are written.
-      #     Deprecated in Elasticsearch 7.0.0.
-      #     Default: 'log'
-      #
-      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
-      #     Override the log level for this appender.
-      #     Default: SemanticLogger.default_level
-      #
-      #   formatter: [Object|Proc|Symbol|Hash]
-      #     An instance of a class that implements #call, or a Proc to be used to format
-      #     the output from this appender
-      #     Default: :raw_json (See: #call)
-      #
-      #   filter: [Regexp|Proc]
-      #     RegExp: Only include log messages where the class name matches the supplied.
-      #     regular expression. All other messages will be ignored.
-      #     Proc: Only include log messages where the supplied Proc returns true
-      #           The Proc must return true or false.
-      #
-      #   host: [String]
-      #     Name of this host to appear in log messages.
-      #     Default: SemanticLogger.host
-      #
-      #   application: [String]
-      #     Name of this application to appear in log messages.
-      #     Default: SemanticLogger.application
-      #
-      # Elasticsearch Parameters:
-      #   url: [String]
-      #     Fully qualified address to the Elasticsearch service.
-      #     Default: 'http://localhost:9200'
-      #
-      #   hosts: [String|Hash|Array]
-      #     Single host passed as a String or Hash, or multiple hosts
-      #     passed as an Array; `host` or `url` keys are also valid.
-      #     Note:
-      #       :url above is ignored when supplying this option.
-      #
-      #   resurrect_after [Float]
-      #     After how many seconds a dead connection should be tried again.
-      #
-      #   reload_connections [true|false|Integer]
-      #     Reload connections after X requests.
-      #     Default: false
-      #
-      #   randomize_hosts [true|false]
-      #     Shuffle connections on initialization and reload.
-      #     Default: false
-      #
-      #   sniffer_timeout [Integer]
-      #     Timeout for reloading connections in seconds.
-      #     Default: 1
-      #
-      #   retry_on_failure [true|false|Integer]
-      #     Retry X times when request fails before raising and exception.
-      #     Default: false
-      #
-      #   retry_on_status [Array<Number>]
-      #     Retry when specific status codes are returned.
-      #
-      #   reload_on_failure [true|false]
-      #     Reload connections after failure.
-      #     Default: false
-      #
-      #   request_timeout [Integer]
-      #     The request timeout to be passed to transport in options.
-      #
-      #   adapter [Symbol]
-      #     A specific adapter for Faraday (e.g. `:patron`)
-      #
-      #   transport_options [Hash]
-      #     Options to be passed to the `Faraday::Connection` constructor.
-      #
-      #   transport_class [Constant]
-      #     A specific transport class to use, will be initialized by
-      #     the client and passed hosts and all arguments.
-      #
-      #   transport [Object]
-      #     A specific transport instance.
-      #
-      #   serializer_class [Constant]
-      #     A specific serializer class to use, will be initialized by
-      #     the transport and passed the transport instance.
-      #
-      #   selector [Elasticsearch::Transport::Transport::Connections::Selector::Base]
-      #     An instance of selector strategy derived from `Elasticsearch::Transport::Transport::Connections::Selector::Base`.
-      #
-      #   send_get_body_as [String]
-      #     Specify the HTTP method to use for GET requests with a body.
-      #     Default: 'GET'
-      def initialize(url: "http://localhost:9200",
-                     index: "semantic_logger",
-                     date_pattern: "%Y.%m.%d",
-                     type: "log",
-                     level: nil,
-                     formatter: nil,
-                     filter: nil,
-                     application: nil,
-                     environment: nil,
-                     host: nil,
-                     data_stream: false,
-                     **elasticsearch_args,
-                     &block)
-        @url                         = url
-        @index                       = index
-        @date_pattern                = date_pattern
-        @type                        = type
-        @elasticsearch_args          = elasticsearch_args.dup
-        @elasticsearch_args[:url]    = url if url && !elasticsearch_args[:hosts]
-        @elasticsearch_args[:logger] = logger
-        @data_stream                 = data_stream
-
-        super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: false, &block)
-        reopen
-      end
-
-      def reopen
-        @client = ::Elasticsearch::Client.new(@elasticsearch_args)
-      end
-
-      # Log to the index for today
-      def log(log)
-        bulk_payload = formatter.call(log, self)
-        write_to_elasticsearch([bulk_index(log), bulk_payload])
-        true
-      end
-
-      def batch(logs)
-        messages = []
-        logs.each do |log|
-          messages << bulk_index(log) << formatter.call(log, self)
-        end
-
-        write_to_elasticsearch(messages)
-        true
-      end
+    class Elasticsearch < ElasticsearchBase
+      # Backwards compatible accessor for the client arguments.
+      alias elasticsearch_args client_args
 
       private
 
-      def write_to_elasticsearch(messages)
-        bulk_result =
-          if @data_stream
-            @client.bulk(index: index, body: messages)
-          else
-            @client.bulk(body: messages)
-          end
-
-        return unless bulk_result["errors"]
-
-        failed = bulk_result["items"].reject { |x| x["status"] == 201 }
-        logger.error("ElasticSearch: Write failed. Messages discarded. : #{failed}")
-      end
-
-      def bulk_index(log)
-        expanded_index_name = log.time.strftime("#{index}-#{date_pattern}")
-        return {"create" => {}} if @data_stream
-
-        bulk_index = {"index" => {"_index" => expanded_index_name}}
-        bulk_index["index"].merge!({"_type" => type}) if version_supports_type?
-        bulk_index
-      end
-
-      def default_formatter
-        time_key = @data_stream ? "@timestamp" : :timestamp
-        SemanticLogger::Formatters::Raw.new(time_format: :iso_8601, time_key: time_key)
+      def client_class
+        ::Elasticsearch::Client
       end
 
       def version_supports_type?

data/lib/semantic_logger/appender/elasticsearch_base.rb

@@ -0,0 +1,212 @@
+require "date"
+
+module SemanticLogger
+  module Appender
+    # Abstract base appender for Elasticsearch-compatible search engines.
+    #
+    # Implements the shared bulk-indexing pipeline used by both the
+    # {Elasticsearch} and {OpenSearch} appenders. Subclasses only need to
+    # supply the backing client class (and, optionally, whether the server
+    # version still supports document `_type`).
+    #
+    # This class is internal: applications add an appender via
+    # `SemanticLogger.add_appender(appender: :elasticsearch, ...)` or
+    # `appender: :opensearch`, never by referencing this class directly.
+    class ElasticsearchBase < SemanticLogger::Subscriber
+      attr_accessor :url, :index, :date_pattern, :type, :client, :flush_interval, :timeout_interval, :batch_size,
+                    :client_args
+
+      # Create an Elasticsearch-compatible appender over persistent HTTP(S).
+      #
+      # Parameters:
+      #   index: [String]
+      #     Prefix of the index to store the logs in.
+      #     The final index appends the date so that indexes are used per day.
+      #       I.e. The final index will look like 'semantic_logger-YYYY.MM.DD'
+      #     Default: 'semantic_logger'
+      #
+      #   date_pattern: [String]
+      #     The time format used to generate the full index name. Useful
+      #       if you want monthly indexes ('%Y.%m') or weekly ('%Y.%W').
+      #     Default: '%Y.%m.%d'
+      #
+      #   type: [String]
+      #     Document type to associate with logs when they are written.
+      #     Deprecated in Elasticsearch 7.0.0, unused by OpenSearch.
+      #     Default: 'log'
+      #
+      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: SemanticLogger.default_level
+      #
+      #   formatter: [Object|Proc|Symbol|Hash]
+      #     An instance of a class that implements #call, or a Proc to be used to format
+      #     the output from this appender
+      #     Default: :raw_json (See: #call)
+      #
+      #   filter: [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied.
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
+      #
+      #   host: [String]
+      #     Name of this host to appear in log messages.
+      #     Default: SemanticLogger.host
+      #
+      #   application: [String]
+      #     Name of this application to appear in log messages.
+      #     Default: SemanticLogger.application
+      #
+      # Client Parameters (passed through to the backing client):
+      #   url: [String]
+      #     Fully qualified address to the service.
+      #     Default: 'http://localhost:9200'
+      #
+      #   hosts: [String|Hash|Array]
+      #     Single host passed as a String or Hash, or multiple hosts
+      #     passed as an Array; `host` or `url` keys are also valid.
+      #     Note:
+      #       :url above is ignored when supplying this option.
+      #
+      #   resurrect_after [Float]
+      #     After how many seconds a dead connection should be tried again.
+      #
+      #   reload_connections [true|false|Integer]
+      #     Reload connections after X requests.
+      #     Default: false
+      #
+      #   randomize_hosts [true|false]
+      #     Shuffle connections on initialization and reload.
+      #     Default: false
+      #
+      #   sniffer_timeout [Integer]
+      #     Timeout for reloading connections in seconds.
+      #     Default: 1
+      #
+      #   retry_on_failure [true|false|Integer]
+      #     Retry X times when request fails before raising and exception.
+      #     Default: false
+      #
+      #   retry_on_status [Array<Number>]
+      #     Retry when specific status codes are returned.
+      #
+      #   reload_on_failure [true|false]
+      #     Reload connections after failure.
+      #     Default: false
+      #
+      #   request_timeout [Integer]
+      #     The request timeout to be passed to transport in options.
+      #
+      #   adapter [Symbol]
+      #     A specific adapter for Faraday (e.g. `:patron`)
+      #
+      #   transport_options [Hash]
+      #     Options to be passed to the `Faraday::Connection` constructor.
+      #
+      #   transport_class [Constant]
+      #     A specific transport class to use, will be initialized by
+      #     the client and passed hosts and all arguments.
+      #
+      #   transport [Object]
+      #     A specific transport instance.
+      #
+      #   serializer_class [Constant]
+      #     A specific serializer class to use, will be initialized by
+      #     the transport and passed the transport instance.
+      #
+      #   selector
+      #     An instance of a connection selector strategy.
+      #
+      #   send_get_body_as [String]
+      #     Specify the HTTP method to use for GET requests with a body.
+      #     Default: 'GET'
+      def initialize(url: "http://localhost:9200",
+                     index: "semantic_logger",
+                     date_pattern: "%Y.%m.%d",
+                     type: "log",
+                     level: nil,
+                     formatter: nil,
+                     filter: nil,
+                     application: nil,
+                     environment: nil,
+                     host: nil,
+                     data_stream: false,
+                     **client_args,
+                     &)
+        @url                  = url
+        @index                = index
+        @date_pattern         = date_pattern
+        @type                 = type
+        @client_args          = client_args.dup
+        @client_args[:url]    = url if url && !client_args[:hosts]
+        @client_args[:logger] = logger
+        @data_stream          = data_stream
+
+        super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: false, &)
+        reopen
+      end
+
+      def reopen
+        @client = client_class.new(@client_args)
+      end
+
+      # Log to the index for today
+      def log(log)
+        bulk_payload = formatter.call(log, self)
+        bulk_write([bulk_index(log), bulk_payload])
+        true
+      end
+
+      def batch(logs)
+        messages = []
+        logs.each do |log|
+          messages << bulk_index(log) << formatter.call(log, self)
+        end
+
+        bulk_write(messages)
+        true
+      end
+
+      private
+
+      # The backing client class, e.g. ::Elasticsearch::Client or ::OpenSearch::Client.
+      def client_class
+        raise NotImplementedError, "#{self.class} must implement #client_class"
+      end
+
+      def bulk_write(messages)
+        bulk_result =
+          if @data_stream
+            @client.bulk(index: index, body: messages)
+          else
+            @client.bulk(body: messages)
+          end
+
+        return unless bulk_result["errors"]
+
+        failed = bulk_result["items"].reject { |x| x["status"] == 201 }
+        logger.error("#{self.class.name}: Write failed. Messages discarded. : #{failed}")
+      end
+
+      def bulk_index(log)
+        expanded_index_name = log.time.strftime("#{index}-#{date_pattern}")
+        return {"create" => {}} if @data_stream
+
+        bulk_index = {"index" => {"_index" => expanded_index_name}}
+        bulk_index["index"].merge!({"_type" => type}) if version_supports_type?
+        bulk_index
+      end
+
+      def default_formatter
+        time_key = @data_stream ? "@timestamp" : :timestamp
+        SemanticLogger::Formatters::Raw.new(time_format: :iso_8601, time_key: time_key)
+      end
+
+      # Modern Elasticsearch (>= 7) and OpenSearch no longer support document `_type`.
+      def version_supports_type?
+        false
+      end
+    end
+  end
+end

data/lib/semantic_logger/appender/elasticsearch_http.rb

@@ -55,10 +55,10 @@ module SemanticLogger
                      type: "log",
                      url: "http://localhost:9200",
                      **http_args,
-                     &block)
+                     &)
         @index = index
         @type  = type
-        super(url: url, **http_args, &block)
+        super(url: url, **http_args, &)
 
         @request_path = "#{@path.end_with?('/') ? @path : "#{@path}/"}#{@index}-%Y.%m.%d"
         @logging_path = "#{@request_path}/#{type}"

data/lib/semantic_logger/appender/file.rb

@@ -7,7 +7,7 @@ module SemanticLogger
   module Appender
     class File < SemanticLogger::Subscriber
       attr_accessor :file_name, :retry_count, :append, :exclusive_lock, :encoding,
-                    :reopen_period, :reopen_count, :reopen_size
+                    :reopen_period, :reopen_count, :reopen_size, :permissions
       attr_reader :log_count, :log_size, :current_file_name, :reopen_at
 
       # Create an appender to log to a named file.
@@ -73,6 +73,13 @@ module SemanticLogger
       #     Encoding to use when writing to the file.
       #     Default: Encoding::BINARY
       #
+      #   :permissions [Integer]
+      #     Octal file permissions to apply to the log file.
+      #     Log files frequently contain sensitive information, so restrict access
+      #     by supplying for example `0o640` (owner read/write, group read).
+      #     Applied both when the file is created and to an existing log file.
+      #     Default: nil (use the process umask, the standard Ruby behavior)
+      #
       #   :retry_count [Integer]
       #     Number of times to attempt to re-open the file name when an error occurs trying to
       #     write to the file.
@@ -123,7 +130,7 @@ module SemanticLogger
       #    logger = SemanticLogger["test"]
       #    logger.info "Hello World"
       def initialize(file_name, retry_count: 1, append: true, reopen_period: nil, reopen_count: 0, reopen_size: 0,
-                     encoding: Encoding::BINARY, exclusive_lock: false, **args, &block)
+                     encoding: Encoding::BINARY, exclusive_lock: false, permissions: nil, **args, &)
         if !file_name.is_a?(String) || file_name.empty?
           raise(ArgumentError, "SemanticLogging::Appender::File file_name must be a non-empty string")
         end
@@ -137,11 +144,12 @@ module SemanticLogger
         @reopen_size    = reopen_size
         @encoding       = encoding
         @exclusive_lock = exclusive_lock
+        @permissions    = permissions
         @log_count      = 0
         @log_size       = 0
         @reopen_at      = nil
 
-        super(**args, &block)
+        super(**args, &)
       end
 
       # After forking an active process call #reopen to re-open
@@ -170,7 +178,15 @@ module SemanticLogger
 
         options = ::File::WRONLY | ::File::CREAT
         options |= ::File::APPEND if append
-        @file = ::File.open(current_file_name, options)
+        @file =
+          if permissions
+            ::File.open(current_file_name, options, permissions)
+          else
+            ::File.open(current_file_name, options)
+          end
+        # File.open only applies the permissions when creating the file, so also
+        # enforce them on an already existing log file.
+        @file.chmod(permissions) if permissions
         # Force all log entries to write immediately without buffering
         # Allows multiple processes to write to the same log file simultaneously
         @file.sync = true

data/lib/semantic_logger/appender/graylog.rb

@@ -88,13 +88,13 @@ module SemanticLogger
                      gelf_options: {},
                      level_map: LevelMap.new,
                      **args,
-                     &block)
+                     &)
         @url          = url
         @max_size     = max_size
         @gelf_options = gelf_options
         @level_map    = level_map.is_a?(LevelMap) ? level_map : LevelMap.new(level_map)
 
-        super(**args, &block)
+        super(**args, &)
         reopen
       end
 

data/lib/semantic_logger/appender/http.rb

@@ -16,6 +16,16 @@ require "openssl"
 #     appender: :http,
 #     url:      'http://localhost:8088/path'
 #   )
+#
+# Batching:
+# By default each log message is posted individually. To post multiple log
+# messages in a single request as a JSON array (for example to the Filebeat
+# http_endpoint input), enable batching when adding the appender:
+#   SemanticLogger.add_appender(
+#     appender: :http,
+#     url:      'http://localhost:8088/path',
+#     batch:    true
+#   )
 module SemanticLogger
   module Appender
     class Http < SemanticLogger::Subscriber
@@ -106,7 +116,7 @@ module SemanticLogger
                      read_timeout: 1.0,
                      continue_timeout: 1.0,
                      **args,
-                     &block)
+                     &)
         @url              = url
         @proxy_url        = proxy_url
         @ssl_options      = ssl
@@ -151,7 +161,7 @@ module SemanticLogger
 
         @http = nil
 
-        super(**args, &block)
+        super(**args, &)
         reopen
       end
 
@@ -191,6 +201,21 @@ module SemanticLogger
         post(message)
       end
 
+      # Forward a batch of log messages to the HTTP Server in a single request.
+      # Only used when the appender is created with `batch: true`.
+      def batch(logs)
+        message = formatter.batch(logs, self)
+        logger.trace(message)
+        post(message)
+      end
+
+      # The HTTP appender supports batching, but only when explicitly requested
+      # via `batch: true`, so that the default behaviour (one request per log
+      # message) is preserved.
+      def batch_by_default?
+        false
+      end
+
       private
 
       # Use JSON Formatter by default

data/lib/semantic_logger/appender/io.rb

@@ -37,13 +37,13 @@ module SemanticLogger
       #
       #    logger = SemanticLogger['test']
       #    logger.info 'Hello World'
-      def initialize(io, **args, &block)
+      def initialize(io, **args, &)
         @io = io
         unless @io.respond_to?(:write)
           raise(ArgumentError, "SemanticLogging::Appender::IO io is not a valid IO instance: #{io.inspect}")
         end
 
-        super(**args, &block)
+        super(**args, &)
       end
 
       def log(log)
@@ -60,8 +60,12 @@ module SemanticLogger
         @io.flush if @io.respond_to?(:flush)
       end
 
-      def console_output?
-        [$stderr, $stdout].include?(@io)
+      def console_stream
+        if @io.equal?($stdout)
+          :stdout
+        elsif @io.equal?($stderr)
+          :stderr
+        end
       end
     end
   end

data/lib/semantic_logger/appender/kafka.rb

@@ -127,7 +127,7 @@ module SemanticLogger
                      ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil, ssl_ca_certs_from_system: false,
                      topic: "log_messages", partition: nil, partition_key: nil, key: nil,
                      delivery_threshold: 100, delivery_interval: 10, required_acks: 1,
-                     metrics: true, **args, &block)
+                     metrics: true, **args, &)
         @seed_brokers             = seed_brokers
         @client_id                = client_id
         @connect_timeout          = connect_timeout
@@ -144,7 +144,7 @@ module SemanticLogger
         @delivery_interval        = delivery_interval
         @required_acks            = required_acks
 
-        super(metrics: metrics, **args, &block)
+        super(metrics: metrics, **args, &)
         reopen
       end
 

data/lib/semantic_logger/appender/loki.rb

@@ -39,14 +39,12 @@ module SemanticLogger
                      header: {"Content-Type" => "application/json"},
                      path: INGESTION_PATH,
                      **args,
-                     &block)
-
-        super(url: "#{url}/#{path}", formatter: formatter, header: header, **args, &block)
+                     &)
+        super(url: "#{url}/#{path}", formatter: formatter, header: header, **args, &)
       end
 
       def log(log)
         message = formatter.call(log, self)
-        puts message
         logger.trace(message)
         post(message)
       end

data/lib/semantic_logger/appender/mongodb.rb

@@ -36,11 +36,8 @@ module SemanticLogger
     # Example:
     #   require 'semantic_logger'
     #
-    #   client   = Mongo::MongoClient.new
-    #   database = client['test']
-    #
     #   appender = SemanticLogger::Appender::MongoDB.new(
-    #     db:              database,
+    #     uri:             'mongodb://127.0.0.1:27017/test',
     #     collection_size: 1024**3 # 1.gigabyte
     #   )
     #   SemanticLogger.add_appender(appender: appender)
@@ -110,7 +107,7 @@ module SemanticLogger
                      collection_size: 1024**3,
                      collection_max: nil,
                      **args,
-                     &block)
+                     &)
         @client          = Mongo::Client.new(uri, logger: logger)
         @collection_name = collection_name
         @options         = {
@@ -125,7 +122,7 @@ module SemanticLogger
         # Create the collection and necessary indexes
         create_indexes
 
-        super(**args, &block)
+        super(**args, &)
       end
 
       # After forking an active process call #reopen to re-open

data/lib/semantic_logger/appender/new_relic_logs.rb

@@ -58,8 +58,8 @@ module SemanticLogger
       def log(log)
         begin
           message = formatter.call(log, self) # Generate the structured log
-          json_message = message.to_json      # Convert the log to JSON
-          level = log.level.to_s.upcase       # Determine the log level
+          json_message = Utils.to_json(message) # Convert the log to JSON, repairing any non UTF-8 data
+          level = log.level.to_s.upcase # Determine the log level
           self.class.log_newrelic(json_message, level)
         rescue JSON::GeneratorError => e
           warn("Failed to serialize log message to JSON: #{e.message}")