semantic_logger 4.18.0 → 5.1.0

data/lib/semantic_logger/appender/cloudwatch_logs.rb

@@ -2,7 +2,8 @@ begin
   require "aws-sdk-cloudwatchlogs"
 rescue LoadError
   raise LoadError,
-        'Gem aws-sdk-cloudwatchlogs is required for logging to CloudWatch Logs. Please add the gem "aws-sdk-cloudwatchlogs" to your Gemfile.'
+        "Gem aws-sdk-cloudwatchlogs is required for logging to CloudWatch Logs. " \
+        'Please add the gem "aws-sdk-cloudwatchlogs" to your Gemfile.'
 end
 
 require "concurrent"
@@ -20,7 +21,8 @@ require "concurrent"
 module SemanticLogger
   module Appender
     class CloudwatchLogs < SemanticLogger::Subscriber
-      attr_reader :client_kwargs, :group, :create_group, :create_stream, :force_flush_interval_seconds, :max_buffered_events,
+      attr_reader :client_kwargs, :group, :create_group, :create_stream,
+                  :force_flush_interval_seconds, :max_buffered_events,
                   :task, :client, :buffered_logs
 
       # Create CloudWatch Logs Appender
@@ -54,7 +56,7 @@ module SemanticLogger
       #     Note that currently CloudWatch Logs has 10000 hard limit.
       #     Default: 4000
       def initialize(
-        *args,
+        *,
         group:,
         client_kwargs: {},
         stream: nil,
@@ -62,8 +64,8 @@ module SemanticLogger
         create_stream: true,
         force_flush_interval_seconds: 5,
         max_buffered_events: 4_000,
-        **kwargs,
-        &block
+        **,
+        &
       )
         @group = group
         @client_kwargs = client_kwargs
@@ -73,7 +75,7 @@ module SemanticLogger
         @force_flush_interval_seconds = force_flush_interval_seconds
         @max_buffered_events = max_buffered_events
 
-        super(*args, **kwargs, &block)
+        super(*, **, &)
         reopen
       end
 

data/lib/semantic_logger/appender/elasticsearch.rb

@@ -2,10 +2,11 @@ begin
   require "elasticsearch"
 rescue LoadError
   raise LoadError,
-        'Gem elasticsearch is required for logging to Elasticsearch. Please add the gem "elasticsearch" to your Gemfile.'
+        "Gem elasticsearch is required for logging to Elasticsearch. " \
+        'Please add the gem "elasticsearch" to your Gemfile.'
 end
 
-require "date"
+require "semantic_logger/appender/elasticsearch_base"
 
 # Forward all log messages to Elasticsearch.
 #
@@ -17,194 +18,14 @@ require "date"
 #   )
 module SemanticLogger
   module Appender
-    class Elasticsearch < SemanticLogger::Subscriber
-      attr_accessor :url, :index, :date_pattern, :type, :client, :flush_interval, :timeout_interval, :batch_size,
-                    :elasticsearch_args
-
-      # Create Elasticsearch appender over persistent HTTP(S)
-      #
-      # Parameters:
-      #   index: [String]
-      #     Prefix of the index to store the logs in Elasticsearch.
-      #     The final index appends the date so that indexes are used per day.
-      #       I.e. The final index will look like 'semantic_logger-YYYY.MM.DD'
-      #     Default: 'semantic_logger'
-      #
-      #   date_pattern: [String]
-      #     The time format used to generate the full index name. Useful
-      #       if you want monthly indexes ('%Y.%m') or weekly ('%Y.%W').
-      #     Default: '%Y.%m.%d'
-      #
-      #   type: [String]
-      #     Document type to associate with logs when they are written.
-      #     Deprecated in Elasticsearch 7.0.0.
-      #     Default: 'log'
-      #
-      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
-      #     Override the log level for this appender.
-      #     Default: SemanticLogger.default_level
-      #
-      #   formatter: [Object|Proc|Symbol|Hash]
-      #     An instance of a class that implements #call, or a Proc to be used to format
-      #     the output from this appender
-      #     Default: :raw_json (See: #call)
-      #
-      #   filter: [Regexp|Proc]
-      #     RegExp: Only include log messages where the class name matches the supplied.
-      #     regular expression. All other messages will be ignored.
-      #     Proc: Only include log messages where the supplied Proc returns true
-      #           The Proc must return true or false.
-      #
-      #   host: [String]
-      #     Name of this host to appear in log messages.
-      #     Default: SemanticLogger.host
-      #
-      #   application: [String]
-      #     Name of this application to appear in log messages.
-      #     Default: SemanticLogger.application
-      #
-      # Elasticsearch Parameters:
-      #   url: [String]
-      #     Fully qualified address to the Elasticsearch service.
-      #     Default: 'http://localhost:9200'
-      #
-      #   hosts: [String|Hash|Array]
-      #     Single host passed as a String or Hash, or multiple hosts
-      #     passed as an Array; `host` or `url` keys are also valid.
-      #     Note:
-      #       :url above is ignored when supplying this option.
-      #
-      #   resurrect_after [Float]
-      #     After how many seconds a dead connection should be tried again.
-      #
-      #   reload_connections [true|false|Integer]
-      #     Reload connections after X requests.
-      #     Default: false
-      #
-      #   randomize_hosts [true|false]
-      #     Shuffle connections on initialization and reload.
-      #     Default: false
-      #
-      #   sniffer_timeout [Integer]
-      #     Timeout for reloading connections in seconds.
-      #     Default: 1
-      #
-      #   retry_on_failure [true|false|Integer]
-      #     Retry X times when request fails before raising and exception.
-      #     Default: false
-      #
-      #   retry_on_status [Array<Number>]
-      #     Retry when specific status codes are returned.
-      #
-      #   reload_on_failure [true|false]
-      #     Reload connections after failure.
-      #     Default: false
-      #
-      #   request_timeout [Integer]
-      #     The request timeout to be passed to transport in options.
-      #
-      #   adapter [Symbol]
-      #     A specific adapter for Faraday (e.g. `:patron`)
-      #
-      #   transport_options [Hash]
-      #     Options to be passed to the `Faraday::Connection` constructor.
-      #
-      #   transport_class [Constant]
-      #     A specific transport class to use, will be initialized by
-      #     the client and passed hosts and all arguments.
-      #
-      #   transport [Object]
-      #     A specific transport instance.
-      #
-      #   serializer_class [Constant]
-      #     A specific serializer class to use, will be initialized by
-      #     the transport and passed the transport instance.
-      #
-      #   selector [Elasticsearch::Transport::Transport::Connections::Selector::Base]
-      #     An instance of selector strategy derived from `Elasticsearch::Transport::Transport::Connections::Selector::Base`.
-      #
-      #   send_get_body_as [String]
-      #     Specify the HTTP method to use for GET requests with a body.
-      #     Default: 'GET'
-      def initialize(url: "http://localhost:9200",
-                     index: "semantic_logger",
-                     date_pattern: "%Y.%m.%d",
-                     type: "log",
-                     level: nil,
-                     formatter: nil,
-                     filter: nil,
-                     application: nil,
-                     environment: nil,
-                     host: nil,
-                     data_stream: false,
-                     **elasticsearch_args,
-                     &block)
-        @url                         = url
-        @index                       = index
-        @date_pattern                = date_pattern
-        @type                        = type
-        @elasticsearch_args          = elasticsearch_args.dup
-        @elasticsearch_args[:url]    = url if url && !elasticsearch_args[:hosts]
-        @elasticsearch_args[:logger] = logger
-        @data_stream                 = data_stream
-
-        super(level: level, formatter: formatter, filter: filter, application: application, environment: environment, host: host, metrics: false, &block)
-        reopen
-      end
-
-      def reopen
-        @client = ::Elasticsearch::Client.new(@elasticsearch_args)
-      end
-
-      # Log to the index for today
-      def log(log)
-        bulk_payload = formatter.call(log, self)
-        write_to_elasticsearch([bulk_index(log), bulk_payload])
-        true
-      end
-
-      def batch(logs)
-        messages = []
-        logs.each do |log|
-          messages << bulk_index(log) << formatter.call(log, self)
-        end
-
-        write_to_elasticsearch(messages)
-        true
-      end
+    class Elasticsearch < ElasticsearchBase
+      # Backwards compatible accessor for the client arguments.
+      alias elasticsearch_args client_args
 
       private
 
-      def write_to_elasticsearch(messages)
-        bulk_result =
-          if @data_stream
-            @client.bulk(index: index, body: messages)
-          else
-            @client.bulk(body: messages)
-          end
-
-        return unless bulk_result["errors"]
-
-        failed = bulk_result["items"].reject { |x| x["status"] == 201 }
-        logger.error("ElasticSearch: Write failed. Messages discarded. : #{failed}")
-      end
-
-      def bulk_index(log)
-        expanded_index_name = log.time.strftime("#{index}-#{date_pattern}")
-        return {"create" => {}} if @data_stream
-
-        bulk_index = {"index" => {"_index" => expanded_index_name}}
-        bulk_index["index"].merge!({"_type" => type}) if version_supports_type?
-        bulk_index
-      end
-
-      def default_formatter
-        time_key = @data_stream ? "@timestamp" : :timestamp
-        SemanticLogger::Formatters::Raw.new(time_format: :iso_8601, time_key: time_key)
-      end
-
-      def version_supports_type?
-        Gem::Version.new(::Elasticsearch::VERSION) < Gem::Version.new(7)
+      def client_class
+        ::Elasticsearch::Client
       end
     end
   end

data/lib/semantic_logger/appender/elasticsearch_base.rb

@@ -0,0 +1,212 @@
+require "date"
+
+module SemanticLogger
+  module Appender
+    # Abstract base appender for Elasticsearch-compatible search engines.
+    #
+    # Implements the shared bulk-indexing pipeline used by both the
+    # {Elasticsearch} and {OpenSearch} appenders. Subclasses only need to
+    # supply the backing client class.
+    #
+    # This class is internal: applications add an appender via
+    # `SemanticLogger.add_appender(appender: :elasticsearch, ...)` or
+    # `appender: :opensearch`, never by referencing this class directly.
+    class ElasticsearchBase < SemanticLogger::Subscriber
+      attr_accessor :url, :index, :date_pattern, :client, :flush_interval, :timeout_interval, :batch_size,
+                    :client_args
+
+      # Create an Elasticsearch-compatible appender over persistent HTTP(S).
+      #
+      # Parameters:
+      #   index: [String]
+      #     Prefix of the index to store the logs in.
+      #     The final index appends the date so that indexes are used per day.
+      #       I.e. The final index will look like 'semantic_logger-YYYY.MM.DD'
+      #     Default: 'semantic_logger'
+      #
+      #   date_pattern: [String]
+      #     The time format used to generate the full index name. Useful
+      #       if you want monthly indexes ('%Y.%m') or weekly ('%Y.%W').
+      #     Default: '%Y.%m.%d'
+      #
+      #   type: [String]
+      #     Deprecated and ignored. Document `_type` has been unused since
+      #     Elasticsearch 7 and is ignored by OpenSearch. Passing it logs a
+      #     deprecation warning; it will be removed in v6.
+      #
+      #   level: [:trace | :debug | :info | :warn | :error | :fatal]
+      #     Override the log level for this appender.
+      #     Default: SemanticLogger.default_level
+      #
+      #   formatter: [Object|Proc|Symbol|Hash]
+      #     An instance of a class that implements #call, or a Proc to be used to format
+      #     the output from this appender
+      #     Default: :raw_json (See: #call)
+      #
+      #   filter: [Regexp|Proc]
+      #     RegExp: Only include log messages where the class name matches the supplied.
+      #     regular expression. All other messages will be ignored.
+      #     Proc: Only include log messages where the supplied Proc returns true
+      #           The Proc must return true or false.
+      #
+      #   host: [String]
+      #     Name of this host to appear in log messages.
+      #     Default: SemanticLogger.host
+      #
+      #   application: [String]
+      #     Name of this application to appear in log messages.
+      #     Default: SemanticLogger.application
+      #
+      # Client Parameters (passed through to the backing client):
+      #   url: [String]
+      #     Fully qualified address to the service.
+      #     Default: 'http://localhost:9200'
+      #
+      #   hosts: [String|Hash|Array]
+      #     Single host passed as a String or Hash, or multiple hosts
+      #     passed as an Array; `host` or `url` keys are also valid.
+      #     Note:
+      #       :url above is ignored when supplying this option.
+      #
+      #   resurrect_after [Float]
+      #     After how many seconds a dead connection should be tried again.
+      #
+      #   reload_connections [true|false|Integer]
+      #     Reload connections after X requests.
+      #     Default: false
+      #
+      #   randomize_hosts [true|false]
+      #     Shuffle connections on initialization and reload.
+      #     Default: false
+      #
+      #   sniffer_timeout [Integer]
+      #     Timeout for reloading connections in seconds.
+      #     Default: 1
+      #
+      #   retry_on_failure [true|false|Integer]
+      #     Retry X times when request fails before raising and exception.
+      #     Default: false
+      #
+      #   retry_on_status [Array<Number>]
+      #     Retry when specific status codes are returned.
+      #
+      #   reload_on_failure [true|false]
+      #     Reload connections after failure.
+      #     Default: false
+      #
+      #   request_timeout [Integer]
+      #     The request timeout to be passed to transport in options.
+      #
+      #   adapter [Symbol]
+      #     A specific adapter for Faraday (e.g. `:patron`)
+      #
+      #   transport_options [Hash]
+      #     Options to be passed to the `Faraday::Connection` constructor.
+      #
+      #   transport_class [Constant]
+      #     A specific transport class to use, will be initialized by
+      #     the client and passed hosts and all arguments.
+      #
+      #   transport [Object]
+      #     A specific transport instance.
+      #
+      #   serializer_class [Constant]
+      #     A specific serializer class to use, will be initialized by
+      #     the transport and passed the transport instance.
+      #
+      #   selector
+      #     An instance of a connection selector strategy.
+      #
+      #   send_get_body_as [String]
+      #     Specify the HTTP method to use for GET requests with a body.
+      #     Default: 'GET'
+      def initialize(url: "http://localhost:9200",
+                     index: "semantic_logger",
+                     date_pattern: "%Y.%m.%d",
+                     type: nil,
+                     level: nil,
+                     formatter: nil,
+                     filter: nil,
+                     application: nil,
+                     environment: nil,
+                     host: nil,
+                     data_stream: false,
+                     **client_args,
+                     &)
+        if type
+          Kernel.warn(
+            "The Elasticsearch/OpenSearch appender `type:` parameter is deprecated and will be removed in v6. " \
+            "Document `_type` has been unused since Elasticsearch 7 and is ignored by OpenSearch.",
+            category: :deprecated
+          )
+        end
+
+        @url                  = url
+        @index                = index
+        @date_pattern         = date_pattern
+        @client_args          = client_args.dup
+        @client_args[:url]    = url if url && !client_args[:hosts]
+        @client_args[:logger] = logger
+        @data_stream          = data_stream
+
+        super(level: level, formatter: formatter, filter: filter, application: application,
+              environment: environment, host: host, metrics: false, &)
+        reopen
+      end
+
+      def reopen
+        @client = client_class.new(@client_args)
+      end
+
+      # Log to the index for today
+      def log(log)
+        bulk_payload = formatter.call(log, self)
+        bulk_write([bulk_index(log), bulk_payload])
+        true
+      end
+
+      def batch(logs)
+        messages = []
+        logs.each do |log|
+          messages << bulk_index(log) << formatter.call(log, self)
+        end
+
+        bulk_write(messages)
+        true
+      end
+
+      private
+
+      # The backing client class, e.g. ::Elasticsearch::Client or ::OpenSearch::Client.
+      def client_class
+        raise NotImplementedError, "#{self.class} must implement #client_class"
+      end
+
+      def bulk_write(messages)
+        bulk_result =
+          if @data_stream
+            @client.bulk(index: index, body: messages)
+          else
+            @client.bulk(body: messages)
+          end
+
+        return unless bulk_result["errors"]
+
+        failed = bulk_result["items"].reject { |x| x["status"] == 201 }
+        logger.error("#{self.class.name}: Write failed. Messages discarded. : #{failed}")
+      end
+
+      def bulk_index(log)
+        expanded_index_name = log.time.strftime("#{index}-#{date_pattern}")
+        return {"create" => {}} if @data_stream
+
+        {"index" => {"_index" => expanded_index_name}}
+      end
+
+      def default_formatter
+        time_key = @data_stream ? "@timestamp" : :timestamp
+        SemanticLogger::Formatters::Raw.new(time_format: :iso_8601, time_key: time_key)
+      end
+    end
+  end
+end

data/lib/semantic_logger/appender/elasticsearch_http.rb

@@ -13,7 +13,7 @@ require "date"
 module SemanticLogger
   module Appender
     class ElasticsearchHttp < SemanticLogger::Appender::Http
-      attr_accessor :index, :type
+      attr_accessor :index
 
       # Create Elasticsearch appender over persistent HTTP(S)
       #
@@ -25,9 +25,9 @@ module SemanticLogger
       #     Default: 'semantic_logger'
       #
       #   type: [String]
-      #     Document type to associate with logs when they are written.
-      #     Deprecated in Elasticsearch 7.0.0
-      #     Default: 'log'
+      #     Deprecated and ignored. Document `_type` has been unused since
+      #     Elasticsearch 7. Passing it logs a deprecation warning; it will be
+      #     removed in v6.
       #
       #   level: [:trace | :debug | :info | :warn | :error | :fatal]
       #     Override the log level for this appender.
@@ -52,16 +52,23 @@ module SemanticLogger
       #     Name of this application to appear in log messages.
       #     Default: SemanticLogger.application
       def initialize(index: "semantic_logger",
-                     type: "log",
+                     type: nil,
                      url: "http://localhost:9200",
                      **http_args,
-                     &block)
+                     &)
+        if type
+          Kernel.warn(
+            "The Elasticsearch/OpenSearch appender `type:` parameter is deprecated and will be removed in v6. " \
+            "Document `_type` has been unused since Elasticsearch 7 and is ignored by OpenSearch.",
+            category: :deprecated
+          )
+        end
+
         @index = index
-        @type  = type
-        super(url: url, **http_args, &block)
+        super(url: url, **http_args, &)
 
         @request_path = "#{@path.end_with?('/') ? @path : "#{@path}/"}#{@index}-%Y.%m.%d"
-        @logging_path = "#{@request_path}/#{type}"
+        @logging_path = "#{@request_path}/_doc"
       end
 
       # Log to the index for today.

data/lib/semantic_logger/appender/file.rb

@@ -7,7 +7,7 @@ module SemanticLogger
   module Appender
     class File < SemanticLogger::Subscriber
       attr_accessor :file_name, :retry_count, :append, :exclusive_lock, :encoding,
-                    :reopen_period, :reopen_count, :reopen_size
+                    :reopen_period, :reopen_count, :reopen_size, :permissions
       attr_reader :log_count, :log_size, :current_file_name, :reopen_at
 
       # Create an appender to log to a named file.
@@ -73,6 +73,13 @@ module SemanticLogger
       #     Encoding to use when writing to the file.
       #     Default: Encoding::BINARY
       #
+      #   :permissions [Integer]
+      #     Octal file permissions to apply to the log file.
+      #     Log files frequently contain sensitive information, so restrict access
+      #     by supplying for example `0o640` (owner read/write, group read).
+      #     Applied both when the file is created and to an existing log file.
+      #     Default: nil (use the process umask, the standard Ruby behavior)
+      #
       #   :retry_count [Integer]
       #     Number of times to attempt to re-open the file name when an error occurs trying to
       #     write to the file.
@@ -123,7 +130,7 @@ module SemanticLogger
       #    logger = SemanticLogger["test"]
       #    logger.info "Hello World"
       def initialize(file_name, retry_count: 1, append: true, reopen_period: nil, reopen_count: 0, reopen_size: 0,
-                     encoding: Encoding::BINARY, exclusive_lock: false, **args, &block)
+                     encoding: Encoding::BINARY, exclusive_lock: false, permissions: nil, **args, &)
         if !file_name.is_a?(String) || file_name.empty?
           raise(ArgumentError, "SemanticLogging::Appender::File file_name must be a non-empty string")
         end
@@ -137,11 +144,17 @@ module SemanticLogger
         @reopen_size    = reopen_size
         @encoding       = encoding
         @exclusive_lock = exclusive_lock
+        @permissions    = permissions
         @log_count      = 0
         @log_size       = 0
         @reopen_at      = nil
 
-        super(**args, &block)
+        super(**args, &)
+
+        # Open the file at creation time so that misconfiguration (bad directory,
+        # insufficient permissions, ...) raises immediately in SemanticLogger.add_appender,
+        # instead of surfacing later on the appender thread via the internal logger.
+        reopen
       end
 
       # After forking an active process call #reopen to re-open
@@ -170,7 +183,15 @@ module SemanticLogger
 
         options = ::File::WRONLY | ::File::CREAT
         options |= ::File::APPEND if append
-        @file = ::File.open(current_file_name, options)
+        @file =
+          if permissions
+            ::File.open(current_file_name, options, permissions)
+          else
+            ::File.open(current_file_name, options)
+          end
+        # File.open only applies the permissions when creating the file, so also
+        # enforce them on an already existing log file.
+        @file.chmod(permissions) if permissions
         # Force all log entries to write immediately without buffering
         # Allows multiple processes to write to the same log file simultaneously
         @file.sync = true

data/lib/semantic_logger/appender/graylog.rb

@@ -88,13 +88,13 @@ module SemanticLogger
                      gelf_options: {},
                      level_map: LevelMap.new,
                      **args,
-                     &block)
+                     &)
         @url          = url
         @max_size     = max_size
         @gelf_options = gelf_options
         @level_map    = level_map.is_a?(LevelMap) ? level_map : LevelMap.new(level_map)
 
-        super(**args, &block)
+        super(**args, &)
         reopen
       end
 

data/lib/semantic_logger/appender/http.rb

@@ -16,6 +16,16 @@ require "openssl"
 #     appender: :http,
 #     url:      'http://localhost:8088/path'
 #   )
+#
+# Batching:
+# By default each log message is posted individually. To post multiple log
+# messages in a single request as a JSON array (for example to the Filebeat
+# http_endpoint input), enable batching when adding the appender:
+#   SemanticLogger.add_appender(
+#     appender: :http,
+#     url:      'http://localhost:8088/path',
+#     batch:    true
+#   )
 module SemanticLogger
   module Appender
     class Http < SemanticLogger::Subscriber
@@ -106,7 +116,7 @@ module SemanticLogger
                      read_timeout: 1.0,
                      continue_timeout: 1.0,
                      **args,
-                     &block)
+                     &)
         @url              = url
         @proxy_url        = proxy_url
         @ssl_options      = ssl
@@ -151,7 +161,7 @@ module SemanticLogger
 
         @http = nil
 
-        super(**args, &block)
+        super(**args, &)
         reopen
       end
 
@@ -191,6 +201,21 @@ module SemanticLogger
         post(message)
       end
 
+      # Forward a batch of log messages to the HTTP Server in a single request.
+      # Only used when the appender is created with `batch: true`.
+      def batch(logs)
+        message = formatter.batch(logs, self)
+        logger.trace(message)
+        post(message)
+      end
+
+      # The HTTP appender supports batching, but only when explicitly requested
+      # via `batch: true`, so that the default behaviour (one request per log
+      # message) is preserved.
+      def batch_by_default?
+        false
+      end
+
       private
 
       # Use JSON Formatter by default