logstash-logger 0.26.1 → 1.1.0

data/lib/logstash-logger/formatter/cee_syslog.rb

@@ -1,21 +1,18 @@
 module LogStashLogger
   module Formatter
     class CeeSyslog < Cee
-      def call(severity, time, progname, message)
-        @progname = progname
-        super
-      end
-
       private
 
-      def build_facility(host)
-        facility = host.dup
-        facility << " #{@progname}" if @progname
-        facility
+      def frame_event(event, progname)
+        "#{build_header(progname)}:#{format_event(event)}\n"
       end
 
-      def format_event(event)
-        "#{build_facility(event["host".freeze])}:#{super}\n"
+      # The header names this process, never a value taken from the logged
+      # content, and always stays on one line.
+      def build_header(progname)
+        header = HOST['hostname'.freeze].to_s.dup
+        header << " #{progname}" if progname
+        header.gsub(/[[:cntrl:]]/, ' '.freeze)
       end
     end
   end

data/lib/logstash-logger/formatter/json.rb

@@ -4,7 +4,20 @@ module LogStashLogger
       private
 
       def format_event(event)
+        # Proactively check for encoding issues to handle cross-platform differences.
+        # Some Ruby implementations (e.g., JRuby) may not raise exceptions during
+        # JSON encoding but produce malformed output instead.
+        if message_has_encoding_issue?(event)
+          log_error(Encoding::InvalidByteSequenceError.new("Invalid encoding in message"))
+          return force_utf8_encoding(event).to_json
+        end
+
         event.to_json
+      rescue Encoding::UndefinedConversionError,
+             Encoding::InvalidByteSequenceError,
+             JSON::GeneratorError => e
+        log_error(e)
+        force_utf8_encoding(event).to_json
       end
     end
   end

data/lib/logstash-logger/formatter/json_lines.rb

@@ -4,7 +4,20 @@ module LogStashLogger
       private
 
       def format_event(event)
+        # Proactively check for encoding issues to handle cross-platform differences.
+        # Some Ruby implementations (e.g., JRuby) may not raise exceptions during
+        # JSON encoding but produce malformed output instead.
+        if message_has_encoding_issue?(event)
+          log_error(Encoding::InvalidByteSequenceError.new("Invalid encoding in message"))
+          return "#{force_utf8_encoding(event).to_json}\n"
+        end
+
         "#{event.to_json}\n"
+      rescue Encoding::UndefinedConversionError,
+             Encoding::InvalidByteSequenceError,
+             JSON::GeneratorError => e
+        log_error(e)
+        "#{force_utf8_encoding(event).to_json}\n"
       end
     end
   end

data/lib/logstash-logger/formatter.rb

@@ -10,20 +10,20 @@ module LogStashLogger
     autoload :Cee, 'logstash-logger/formatter/cee'
     autoload :CeeSyslog, 'logstash-logger/formatter/cee_syslog'
 
-    def self.new(formatter_type, customize_event: nil)
-      build_formatter(formatter_type, customize_event)
+    def self.new(formatter_type, customize_event: nil, error_logger: LogStashLogger.configuration.default_error_logger)
+      build_formatter(formatter_type, customize_event, error_logger)
     end
 
-    def self.build_formatter(formatter_type, customize_event)
+    def self.build_formatter(formatter_type, customize_event, error_logger)
       formatter_type ||= DEFAULT_FORMATTER
 
       formatter = if custom_formatter_instance?(formatter_type)
         formatter_type
       elsif custom_formatter_class?(formatter_type)
-        formatter_type.new
+        initialize_custom_formatter_klass(formatter_type, customize_event)
       else
-        formatter_klass(formatter_type).new(customize_event: customize_event)
-      end
+        formatter_klass(formatter_type).new(customize_event: customize_event, error_logger: error_logger)
+                  end
 
       formatter.send(:extend, ::LogStashLogger::TaggedLogging::Formatter)
       formatter
@@ -47,5 +47,13 @@ module LogStashLogger
     def self.custom_formatter_class?(formatter_type)
       formatter_type.is_a?(Class) && formatter_type.method_defined?(:call)
     end
+
+    def self.initialize_custom_formatter_klass(formatter_klass, customize_event)
+      if formatter_klass.instance_method(:initialize).parameters.include?([:key, :customize_event])
+        formatter_klass.new(customize_event: customize_event)
+      else
+        formatter_klass.new
+      end
+    end
 	end
 end

data/lib/logstash-logger/logger.rb

@@ -26,31 +26,19 @@ module LogStashLogger
     end
   end
 
-  protected
 
   def self.extract_opts(*args)
-    args.flatten!
-
-    if args.length > 1
-      if args.all?{|arg| arg.is_a?(Hash)}
-        # Deprecated array of hashes
-        warn "[LogStashLogger] Passing an array of hashes to the constructor is deprecated. Please replace with an options hash: { type: :multi_delegator, outputs: [...] }"
-        { type: :multi_delegator, outputs: args }
-      else
-        # Deprecated host/port/type constructor
-        warn "[LogStashLogger] The (host, port, type) constructor is deprecated. Please use an options hash instead."
-        host, port, type = *args
-        { host: host, port: port, type: type }
-      end
-    elsif Hash === args[0]
+    if args.length == 1 && args[0].is_a?(Hash)
       args[0]
     else
-      fail ArgumentError, "Invalid LogStashLogger options"
+      fail ArgumentError, "Invalid LogStashLogger options. Expected a single options hash."
     end
   end
 
   def self.build_logger(opts)
-    formatter = Formatter.new(opts.delete(:formatter), customize_event: opts.delete(:customize_event))
+    formatter = Formatter.new(opts.delete(:formatter),
+                              customize_event: opts.delete(:customize_event),
+                              error_logger: opts.fetch(:error_logger, LogStashLogger.configuration.default_error_logger))
 
     logger_type = opts[:type].to_s.to_sym
     logger = case logger_type
@@ -60,13 +48,12 @@ module LogStashLogger
       build_syslog_logger(opts)
     else
       build_default_logger(opts)
-    end
+             end
 
     logger.formatter = formatter if formatter
     logger
   end
 
-  private
 
   def self.build_default_logger(opts)
     logger_class = opts.delete(:logger_class) || ::Logger

data/lib/logstash-logger/multi_logger.rb

@@ -70,7 +70,7 @@ module LogStashLogger
     def method_missing(name, *args, &block)
       @loggers.each do |logger|
         if logger.respond_to?(name)
-          logger.send(name, args, &block)
+          logger.send(name, *args, &block)
         end
       end
     end
@@ -101,6 +101,7 @@ module LogStashLogger
         logger.add(severity, message, progname, &block)
       end
     end
+    alias log add
 
     def <<(msg)
       @loggers.each do |logger|

data/lib/logstash-logger/railtie.rb

@@ -25,7 +25,7 @@ module LogStashLogger
     # Convert logger options to OrderedOptions if regular Hash
     logger_options = ActiveSupport::OrderedOptions.new.merge(logger_options)
 
-    if parsed_uri_options = LogStashLogger::Device.parse_uri_config(logger_options)
+    if (parsed_uri_options = LogStashLogger::Device.parse_uri_config(logger_options))
       logger_options.delete(:uri)
       logger_options.merge!(parsed_uri_options)
     end

data/lib/logstash-logger/tagged_logging.rb

@@ -19,7 +19,9 @@ module LogStashLogger
       end
 
       def push_tags(*tags)
-        tags.flatten.reject{ |t| t.nil? || t.empty? }.tap do |new_tags|
+        non_empty_tags = tags.flatten.compact.reject { |t| t.respond_to?(:empty?) && t.empty? }
+
+        non_empty_tags.tap do |new_tags|
           current_tags.concat new_tags
         end
       end

data/lib/logstash-logger/version.rb

@@ -1,3 +1,3 @@
 module LogStashLogger
-  VERSION = "0.26.1"
+  VERSION = "1.1.0"
 end

data/logstash-logger.gemspec

@@ -22,13 +22,11 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency 'rails'
   gem.add_development_dependency 'redis'
-  gem.add_development_dependency 'poseidon'
+  gem.add_development_dependency 'ruby-kafka'
   gem.add_development_dependency 'aws-sdk-kinesis'
   gem.add_development_dependency 'aws-sdk-firehose'
 
-  if defined?(JRUBY_VERSION)
-    gem.add_development_dependency 'SyslogLogger'
-  end
+  gem.add_development_dependency 'syslog', '>= 0.4'
 
   gem.add_development_dependency 'rspec', '>= 3'
   gem.add_development_dependency 'rake'
@@ -36,4 +34,10 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'wwtd'
   gem.add_development_dependency 'appraisal'
   gem.add_development_dependency 'rubocop'
+  gem.add_development_dependency 'rubocop-performance'
+  gem.add_development_dependency 'rubocop-rails'
+  gem.add_development_dependency 'rubocop-rake'
+  gem.add_development_dependency 'rubocop-rspec'
+
+  gem.required_ruby_version = '>= 3.3'
 end

data/spec/device/aws_stream_limits_spec.rb

@@ -0,0 +1,219 @@
+require 'logstash-logger'
+require 'logger'
+
+# Stub AWS clients that record every request and can be told to fail.
+class FakeKinesisClient
+  attr_reader :requests
+
+  def initialize(fail_requests: [])
+    @requests = []
+    @fail_requests = fail_requests
+  end
+
+  def put_records(params)
+    @requests << params[:records].map { |r| r[:data] }
+    raise Seahorse::Client::NetworkingError.new(StandardError.new('connection reset')) if @fail_requests.include?(@requests.size)
+    entries = params[:records].map { Aws::Kinesis::Types::PutRecordsResultEntry.new(sequence_number: '1', shard_id: 's') }
+    Aws::Kinesis::Types::PutRecordsOutput.new(failed_record_count: 0, records: entries)
+  end
+end
+
+class FakeFirehoseClient
+  attr_reader :requests
+
+  def initialize
+    @requests = []
+  end
+
+  def put_record_batch(params)
+    @requests << params[:records].map { |r| r[:data] }
+    entries = params[:records].map { Aws::Firehose::Types::PutRecordBatchResponseEntry.new(record_id: '1') }
+    Aws::Firehose::Types::PutRecordBatchOutput.new(failed_put_count: 0, request_responses: entries)
+  end
+end
+
+describe 'AWS outputs and service limits' do
+  let(:error_log) { StringIO.new }
+  let(:common) do
+    { sync: false, buffer_max_interval: 3600, buffer_flush_at_exit: false,
+      aws_region: 'us-east-1', error_logger: ::Logger.new(error_log) }
+  end
+  let(:small) { (1..5).map { |i| %({"n":#{i}}) } }
+
+  def pending_count(device)
+    device.instance_variable_get(:@buffer_state)[:pending_count]
+  end
+
+  describe LogStashLogger::Device::Kinesis do
+    let(:client) { FakeKinesisClient.new }
+    let(:device) { LogStashLogger::Device.new(common.merge(type: :kinesis)) }
+
+    before { allow(Aws::Kinesis::Client).to receive(:new) { client } }
+
+    it 'drops only a record over the per-record limit' do
+      device.write_batch(small + ['x' * 1_048_576])
+      expect(client.requests).to eq([small])
+      expect(error_log.string).to match(/dropping a record of 1048612 bytes: over the 1048576-byte limit/)
+    end
+
+    it 'accepts larger records when aws_max_record_bytes is raised' do
+      big = 'x' * 2_000_000
+      device = LogStashLogger::Device.new(common.merge(type: :kinesis, aws_max_record_bytes: 10_485_760))
+      device.write_batch([big])
+      expect(client.requests).to eq([[big]])
+    end
+
+    it 'splits more than 500 records into several requests' do
+      messages = (1..600).map(&:to_s)
+      device.write_batch(messages)
+      expect(client.requests.map(&:size)).to eq([500, 100])
+    end
+
+    it 'splits requests that would go over the request-size limit' do
+      messages = Array.new(12) { 'x' * 1_000_000 }
+      device.write_batch(messages)
+      expect(client.requests.map(&:size)).to eq([10, 2])
+    end
+
+    it 'sends records that are not Strings as text' do
+      device.write_batch([123, :sym])
+      expect(client.requests).to eq([['123', 'sym']])
+    end
+
+    context 'when a request fails' do
+      let(:client) { FakeKinesisClient.new(fail_requests: [1]) }
+
+      it 'keeps the records for the next flush instead of treating them as delivered' do
+        small.each { |m| device.write(m) }
+        expect { device.buffer_flush(force: true) }.not_to raise_error
+        expect(pending_count(device)).to eq(small.size)
+        expect(error_log.string).to include('5 records will be retried on the next flush')
+      end
+
+      it 'drops them when drop_messages_on_flush_error is set' do
+        device = LogStashLogger::Device.new(common.merge(type: :kinesis, drop_messages_on_flush_error: true))
+        small.each { |m| device.write(m) }
+        device.buffer_flush(force: true)
+        expect(pending_count(device)).to eq(0)
+        expect(error_log.string).to include('dropping 5 records after a failed request')
+      end
+    end
+
+    context 'when a later request fails' do
+      let(:client) { FakeKinesisClient.new(fail_requests: [2]) }
+
+      it 'retries only the records that were not sent' do
+        device.write_batch((1..600).map(&:to_s))
+        expect(pending_count(device)).to eq(100)
+      end
+    end
+
+    it 'delivers the rest of a buffered batch around an oversized record' do
+      small.each { |m| device.write(m) }
+      device.write('x' * 2_000_000)
+      device.buffer_flush(force: true)
+      expect(client.requests).to eq([small])
+      expect(pending_count(device)).to eq(0)
+    end
+  end
+
+  describe LogStashLogger::Device::Firehose do
+    let(:client) { FakeFirehoseClient.new }
+    let(:device) { LogStashLogger::Device.new(common.merge(type: :firehose)) }
+
+    before { allow(Aws::Firehose::Client).to receive(:new) { client } }
+
+    it 'drops only a record over the 1,000 KiB limit' do
+      device.write_batch(small + ['x' * 1_024_001])
+      expect(client.requests).to eq([small])
+    end
+
+    it 'splits requests at 4 MB' do
+      device.write_batch(Array.new(5) { 'x' * 1_000_000 })
+      expect(client.requests.map(&:size)).to eq([4, 1])
+    end
+  end
+end
+
+# Throttles every record of the first `throttled` requests, then accepts all.
+class ThrottlingKinesisClient
+  attr_reader :requests
+
+  def initialize(throttled:, code: 'ProvisionedThroughputExceededException')
+    @requests = []
+    @throttled = throttled
+    @code = code
+  end
+
+  def put_records(params)
+    @requests << params[:records].map { |r| r[:data] }
+    if @requests.size <= @throttled
+      entries = params[:records].map { Aws::Kinesis::Types::PutRecordsResultEntry.new(error_code: @code, error_message: 'slow down') }
+      Aws::Kinesis::Types::PutRecordsOutput.new(failed_record_count: entries.size, records: entries)
+    else
+      entries = params[:records].map { Aws::Kinesis::Types::PutRecordsResultEntry.new(sequence_number: '1', shard_id: 's') }
+      Aws::Kinesis::Types::PutRecordsOutput.new(failed_record_count: 0, records: entries)
+    end
+  end
+
+  def delivered
+    @requests.drop(@throttled).flatten
+  end
+end
+
+describe LogStashLogger::Device::Kinesis, 'when records are throttled' do
+  let(:error_log) { StringIO.new }
+  let(:client) { ThrottlingKinesisClient.new(throttled: 1) }
+  let(:records) { (1..50).map(&:to_s) }
+
+  before { allow(Aws::Kinesis::Client).to receive(:new) { client } }
+
+  def device(**opts)
+    LogStashLogger::Device.new({ type: :kinesis, aws_region: 'us-east-1', sync: false, buffer_max_interval: 0.2,
+                                 buffer_flush_at_exit: false, error_logger: ::Logger.new(error_log) }.merge(opts))
+  end
+
+  # These flushes run on the buffer's background timer thread, so poll until
+  # the block returns true, giving up after `timeout` seconds. The example's
+  # own expectations then report what went wrong if it never does.
+  def wait_until(timeout: 5)
+    deadline = Time.now + timeout
+    sleep 0.05 until yield || Time.now > deadline
+  end
+
+  it 'retries a full throttled batch on the timer thread without losing it or stopping the timer' do
+    dev = device
+    timer = dev.instance_variable_get(:@flush_timer_thread)
+    records.each { |r| dev.write(r) }
+
+    wait_until { client.delivered.size == records.size }
+
+    expect(client.delivered).to match_array(records)
+    expect(dev.instance_variable_get(:@buffer_state)[:pending_count]).to eq(0)
+    expect(dev.instance_variable_get(:@buffer_state)[:outgoing_count]).to eq(0)
+    expect(timer).to be_alive
+    expect(error_log.string).to include('50 records will be retried on the next flush (ProvisionedThroughputExceededException: 50)')
+    expect(error_log.string).not_to include('Buffer Full')
+  end
+
+  it 'does not block logging when drop_messages_on_full_buffer is false' do
+    dev = device(drop_messages_on_full_buffer: false)
+    records.each { |r| dev.write(r) }
+    wait_until { client.delivered.size == records.size }
+
+    writer = Thread.new { dev.write('after') }
+    expect(writer.join(2)).not_to be_nil
+    expect(client.delivered).to match_array(records)
+  end
+
+  context 'with InternalFailure' do
+    let(:client) { ThrottlingKinesisClient.new(throttled: 1, code: 'InternalFailure') }
+
+    it 'retries the records' do
+      dev = device
+      records.first(3).each { |r| dev.write(r) }
+      wait_until { client.delivered.size == 3 }
+      expect(client.delivered).to match_array(records.first(3))
+    end
+  end
+end

data/spec/device/connectable_spec.rb

@@ -72,3 +72,54 @@ describe LogStashLogger::Device::Connectable do
     end
   end
 end
+
+describe LogStashLogger::Device::Connectable, 'max_record_bytes' do
+  let(:error_log) { StringIO.new }
+
+  def device(**opts)
+    LogStashLogger::Device.new({ type: :udp, port: 5228, sync: false, buffer_max_interval: 3600,
+                                 buffer_flush_at_exit: false, error_logger: ::Logger.new(error_log) }.merge(opts))
+  end
+
+  def pending(dev)
+    dev.instance_variable_get(:@buffer_state)[:pending_items][nil]
+  end
+
+  it 'is off by default' do
+    dev = device
+    dev.write('x' * 2_000_000)
+    expect(pending(dev).size).to eq(1)
+  end
+
+  it 'drops only records over the limit, logging the size but not the content' do
+    dev = device(max_record_bytes: 100)
+    dev.write('small')
+    dev.write('secret' * 50)
+    expect(pending(dev)).to eq(['small'])
+    expect(error_log.string).to include('dropping a record of 300 bytes: over max_record_bytes (100)')
+    expect(error_log.string).not_to include('secret')
+  end
+
+  it 'measures records that are not Strings' do
+    dev = device(max_record_bytes: 2)
+    dev.write(123)
+    expect(pending(dev)).to be_empty
+  end
+
+  it 'accepts a String value, as URI query options arrive' do
+    dev = LogStashLogger::Device.new(uri: 'udp://localhost:5228?max_record_bytes=100', buffer_flush_at_exit: false,
+                                     error_logger: ::Logger.new(error_log))
+    dev.write('x' * 101)
+    expect(pending(dev)).to be_empty
+  end
+
+  it 'rejects a value that is not a positive number' do
+    expect { device(max_record_bytes: 'lots') }.to raise_error(ArgumentError)
+    expect { device(max_record_bytes: 0) }.to raise_error(ArgumentError, /positive/)
+  end
+
+  it 'is not forwarded to the Redis client' do
+    expect(::Redis).to receive(:new).with(hash_excluding(:max_record_bytes)).and_return(double('redis'))
+    LogStashLogger::Device.new(type: :redis, max_record_bytes: 100, buffer_flush_at_exit: false).connect
+  end
+end

data/spec/device/file_spec.rb

@@ -7,6 +7,71 @@ describe LogStashLogger::Device::File do
     expect(file_device.to_io).to be_a ::File
   end
 
+  context "with shift options" do
+    let(:shift_file) do
+      temp = Tempfile.new('logstash_shift')
+      temp.close
+      temp
+    end
+
+    let(:shift_device) do
+      LogStashLogger::Device.new(
+        type: :file,
+        path: shift_file.path,
+        shift_age: 1,
+        shift_size: 1
+      )
+    end
+
+    let(:period_file) do
+      temp = Tempfile.new('logstash_period_shift')
+      temp.close
+      temp
+    end
+
+    let(:period_device) do
+      LogStashLogger::Device.new(
+        type: :file,
+        path: period_file.path,
+        shift_age: 'daily',
+        shift_period_suffix: '%Y-%m-%d'
+      )
+    end
+
+    after do
+      shift_device.close
+      ::Dir.glob("#{shift_file.path}*").each do |path|
+        ::File.delete(path) if ::File.exist?(path)
+      end
+
+      period_device.close
+      ::Dir.glob("#{period_file.path}*").each do |path|
+        ::File.delete(path) if ::File.exist?(path)
+      end
+    end
+
+    it "wraps a Logger::LogDevice for rotation" do
+      expect(shift_device.io).to be_a ::Logger::LogDevice
+      expect(shift_device.to_io).to be_a ::File
+    end
+
+    it "rotates when shift_size is exceeded" do
+      shift_device.write("a" * 10)
+      shift_device.write("b" * 10)
+      shift_device.close
+
+      expect(::File.exist?("#{shift_file.path}.0")).to be(true)
+    end
+
+    it "uses shift_period_suffix for time-based rotation" do
+      period_device.write("a")
+      period_device.io.send(:shift_log_period, Time.new(2026, 1, 22))
+      period_device.close
+
+      expect(::File.exist?("#{period_file.path}.2026-01-22")).to be(true)
+    end
+  end
+
   context "when path is not specified" do
     it "raises an exception" do
       expect { described_class.new }.to raise_error(ArgumentError)

data/spec/device/firehose_spec.rb

@@ -30,9 +30,12 @@ describe LogStashLogger::Device::Firehose do
     response.request_responses = [failed_record]
 
     expect(client).to receive(:put_record_batch) { response }
-    expect(firehose_device).to receive(:write).with("foo")
+    expect(firehose_device).not_to receive(:write)
 
     firehose_device.write_one "foo"
+    state = firehose_device.instance_variable_get(:@buffer_state)
+    expect(state[:pending_items][nil]).to eq(["foo"])
+    expect(state[:pending_count]).to eq(1)
   end
 
   it "defaults the Firehose stream to logstash" do