logstash-logger 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec434ced077d0200f2735e5e2f0ccfb54c5969f5b8790f68ce8178b472d60833
-  data.tar.gz: 56bb45aab80a9640c7fea253aad346f3db7d1adf2194c0f95721a1f61229fe3b
+  metadata.gz: 4f311cb07a6b81dd87a40cd78b4bbfba8742575dcfc888f1cfa85e647b7512c4
+  data.tar.gz: ce6a47b4dce5299041b138e97b4923d67a7efaf7816352135a049ab044d90843
 SHA512:
-  metadata.gz: e6dc9be621c4710115af8fe97c23983c2f4bdebbaa865db38334f02b5f27bd56ec639032c1d5696788b650879b16d758370a8de3684d0763df36f73a36d99066
-  data.tar.gz: f1c642f5c57118087142aaed68cc760cfc769646f7168baa1558048f0ae0b2665024e8deb09d15086c4e1a247a34b6be78abe13341f2e46af34e42921b71910f
+  metadata.gz: c91bfa3f1b0e569c2744fdce29fbd6097021c0f552e743139aeb98af818ead41eae66bdc2290e353c8a6213b1ec7556f325deaf254f7fe92c3e3589305ee7a6b
+  data.tar.gz: db8c4ddee010de0876eb2eb9066d8faf0c2fc8604ae7bc1655526d9c7a86f2bd4fffea02267dfdaa3dd1acbd820479bf50cf3ed9e00b92fcbdb3c724e251881d

data/.github/workflows/tests.yml

@@ -16,9 +16,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ['3.2', '3.3', '3.4', '4.0', 'jruby-10']
-        rails: ['7.2', '8.0', '8.1']
+        ruby: ['3.3', '3.4', '4.0', 'jruby-10']
+        rails: ['8.0', '8.1']
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       
@@ -44,8 +45,9 @@ jobs:
           bundle exec appraisal rails-${{ matrix.rails }} rake default
       
       - name: Upload coverage to Codecov
-        if: matrix.ruby == '3.2' && matrix.rails == '8.1'
+        if: matrix.ruby == '3.3' && matrix.rails == '8.1'
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage/coverage.xml
           fail_ci_if_error: false

data/Appraisals

@@ -1,9 +1,7 @@
-appraise "rails-7.2" do
-  gem "rails", "~> 7.2.0"
-end
-
 appraise "rails-8.0" do
   gem "rails", "~> 8.0.0"
+  # ActiveSupport < 8.1 passes quirks_mode to JSON, which json 3.x rejects
+  gem "json", "< 3"
 end
 
 appraise "rails-8.1" do

data/CHANGELOG.md

@@ -1,3 +1,36 @@
+## 1.1.0
+
+### Security
+- The TCP output with `ssl_enable: true` and no `ssl_context` accepted any server certificate, so anyone able to intercept the connection could read or alter the log stream. It now verifies the certificate and hostname. [GHSA-p7gr-ppfj-g45v](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-p7gr-ppfj-g45v)
+- Logged content could break record framing: inject a line break into the `cee_syslog` header, or trigger an unframed `Failed to format log event` string that merged with the next record. [GHSA-88vf-w58m-gv7h](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-88vf-w58m-gv7h)
+- One record that Kafka, Kinesis, Firehose or an HTTP collector always rejects could make the output hold back, duplicate or discard the other records buffered with it. [GHSA-qcm4-6phg-mhpf](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-qcm4-6phg-mhpf)
+
+### Breaking Changes
+- Drop support for Ruby 3.2 and Rails 7.2, which are end of life. [#185](https://github.com/dwbutler/logstash-logger/pull/185)
+- `ssl_enable: true` without an `ssl_context` now verifies the server certificate against the system trust store. For a self-signed or private-CA server, pass its certificate as `ssl_certificate`. Turning verification off requires an explicit `VERIFY_NONE` context. See [SSL Configuration](docs/ssl.md). [GHSA-p7gr-ppfj-g45v](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-p7gr-ppfj-g45v)
+- `ssl_certificate` is now the certificate trusted for the server, as documented, and is checked when the logger is created. [GHSA-p7gr-ppfj-g45v](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-p7gr-ppfj-g45v)
+- The `cee_syslog` header always uses this machine's hostname. A `host` set in logged content or in `customize_event` appears only in the JSON body. [GHSA-88vf-w58m-gv7h](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-88vf-w58m-gv7h)
+
+### Features
+- `format: :ndjson` option for the HTTP device, to send newline-delimited JSON instead of a JSON array, and a `content_type` option. [#182](https://github.com/dwbutler/logstash-logger/pull/182) [#180](https://github.com/dwbutler/logstash-logger/issues/180)
+- `max_record_bytes` option to drop any formatted record over a size limit, for every buffered output. Off by default. [GHSA-qcm4-6phg-mhpf](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-qcm4-6phg-mhpf)
+- `aws_max_record_bytes` option for Kinesis streams configured for records larger than 1 MiB. [GHSA-qcm4-6phg-mhpf](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-qcm4-6phg-mhpf)
+- The TCP output sends SNI for TLS connections to DNS hostnames. [GHSA-p7gr-ppfj-g45v](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-p7gr-ppfj-g45v)
+
+### Bug Fixes
+- HTTP device raises on non-2xx responses, so failed uploads are retried instead of silently dropped. [#184](https://github.com/dwbutler/logstash-logger/pull/184) [#181](https://github.com/dwbutler/logstash-logger/issues/181)
+- HTTP device drops only the records a collector rejects with 400, 413, 415 or 422. Other error statuses are retried. [GHSA-qcm4-6phg-mhpf](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-qcm4-6phg-mhpf)
+- Kinesis and Firehose enforce the service's record and request limits, retry failed requests instead of treating them as delivered, and requeue throttled records without losing the batch. Kinesis `InternalFailure` records are retried. [GHSA-qcm4-6phg-mhpf](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-qcm4-6phg-mhpf)
+- Kafka drops only records the broker always rejects, instead of retrying the whole batch until the buffer fills. [GHSA-qcm4-6phg-mhpf](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-qcm4-6phg-mhpf)
+- The TCP output closes the underlying TCP socket when a TLS connection is closed or fails to set up. [GHSA-p7gr-ppfj-g45v](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-p7gr-ppfj-g45v)
+- A record that cannot be formatted is replaced by a placeholder event tagged `_logstashlogger_format_failure`, framed like any other record. [GHSA-88vf-w58m-gv7h](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-88vf-w58m-gv7h)
+- An `@timestamp` in logged content that is not a valid time no longer fails the record. [GHSA-88vf-w58m-gv7h](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-88vf-w58m-gv7h)
+- The `cee_syslog` formatter no longer shares the program name between concurrent log calls. [GHSA-88vf-w58m-gv7h](https://github.com/dwbutler/logstash-logger/security/advisories/GHSA-88vf-w58m-gv7h)
+
+### Other
+- CI covers Ruby 3.3, 3.4, 4.0 and JRuby 10 with Rails 8.0 and 8.1, and uploads coverage to Codecov. [#183](https://github.com/dwbutler/logstash-logger/pull/183) [#185](https://github.com/dwbutler/logstash-logger/pull/185)
+- Fix an order-dependent Kafka spec. [#186](https://github.com/dwbutler/logstash-logger/pull/186)
+
 ## 1.0.0
 
 ### Breaking Changes

data/Gemfile

@@ -2,4 +2,5 @@ source 'https://rubygems.org'
 
 gemspec
 
-gem "codeclimate-test-reporter", group: :test, require: nil
+gem "simplecov", ">= 0.22", group: :test, require: false
+gem "simplecov-cobertura", group: :test, require: false

data/README.md

@@ -35,10 +35,10 @@ Or install it yourself as:
 
 Verified to work with:
 
-* MRI Ruby 3.2, 3.3, 3.4, 4.0
+* MRI Ruby 3.3, 3.4, 4.0
 * JRuby 10.x
 
-Ruby versions < 3.2 are EOL'ed and no longer supported.
+Ruby versions < 3.3 are EOL'ed and no longer supported.
 
 ## Documentation
 

data/docs/buffering.md

@@ -18,6 +18,7 @@ You can configure buffering behavior by passing the following options to LogStas
 * `:sync` - Flush buffer every time a message is received (blocking). Defaults to false.
 * `:buffer_flush_at_exit` - Flush messages when exiting the program. Defaults to true.
 * `:buffer_logger` - Logger to write buffer debug/error messages to. Defaults to none.
+* `:max_record_bytes` - Drop any single formatted record larger than this many bytes, with a warning that gives its size. Off by default.
 
 You can turn buffering off by setting `sync = true`.
 
@@ -54,6 +55,21 @@ If you don't want to lose messages when the buffer gets full, you can set
 `drop_messages_on_full_buffer = false`. Note that if the buffer gets full, any
 incoming log message will block, which could be undesirable.
 
+## Records that can never be delivered
+
+Some outputs can reject a record for a reason that will not go away on retry, such as its size.
+Instead of retrying the whole batch until the buffer fills, these outputs drop only the rejected
+records, and log their size and the reason, never their content:
+
+* Kafka: records the broker or client rejects with `MessageSizeTooLarge`, `RecordListTooLarge`
+  or `BufferOverflow`. The rest of the batch is sent one record at a time.
+* Kinesis and Firehose: records over the service's per-record limit (Kinesis 1 MiB by default,
+  Firehose 1,000 KiB). Batches are split to fit each service's request limits. If your Kinesis
+  stream is configured for larger records, raise the limit with `aws_max_record_bytes`, for
+  example `aws_max_record_bytes: 10 * 1024 * 1024`.
+* HTTP: records the collector rejects with 400, 413, 415 or 422. The rest of the batch is sent
+  one record at a time. Other error statuses, such as 401, 429 or 5xx, are retried.
+
 ## Sync Mode
 
 All logger outputs support a `sync` setting. This is analogous to the "sync mode" setting on Ruby IO

data/docs/outputs.md

@@ -39,4 +39,27 @@ LogStashLogger.new \
   url: 'http://localhost:8080'
 ```
 
+A batch that gets an error status is retried on the next flush. If the collector rejects the content
+(400, 413, 415 or 422), the records are sent one at a time and only the rejected ones are dropped.
+See [Records that can never be delivered](buffering.md#records-that-can-never-be-delivered).
+
 Note the parameter is `url` and not `uri`. Relies on [Net:HTTP](https://ruby-doc.org/stdlib-2.7.1/libdoc/net/http/rdoc/Net/HTTP.html#class-Net::HTTP-label-HTTPS) to auto-detect SSL usage from the scheme.
+
+By default, each batch is sent as a JSON array, which is what the Logstash HTTP Input expects. For endpoints that expect newline-delimited JSON (NDJSON), with one event per line, set `format: :ndjson`:
+
+```ruby
+LogStashLogger.new \
+  type: :http,
+  url: 'https://example.com:8071/?token=TOKEN',
+  format: :ndjson
+```
+
+The `format` option accepts `:json_array` (the default) or `:ndjson`. With `:ndjson` the request is sent with `Content-Type: application/x-ndjson`; with `:json_array` it uses `application/json`. If your endpoint expects a different header, set it explicitly with `content_type`:
+
+```ruby
+LogStashLogger.new \
+  type: :http,
+  url: 'https://example.com:8071/?token=TOKEN',
+  format: :ndjson,
+  content_type: 'application/json'
+```

data/docs/rails.md

@@ -1,6 +1,6 @@
 # Rails Integration
 
-Supports Rails 7.2, 8.0, and 8.1.
+Supports Rails 8.0 and 8.1.
 
 By default, every Rails log message will be written to logstash in `LogStash::Event` JSON format.
 
@@ -74,10 +74,15 @@ config.logstash.port = 5228
 # Required
 config.logstash.type = :tcp
 
-# Optional, enables SSL
+# Optional, enables SSL and verifies the server certificate against the system trust store
 config.logstash.ssl_enable = true
+
+# Optional, trusts this certificate (for a self-signed or private-CA server) and enables SSL
+# config.logstash.ssl_certificate = '/path/to/certificate.crt'
 ```
 
+See [SSL Configuration](ssl.md) for other options.
+
 ### Unix Socket
 
 ```ruby

data/docs/ssl.md

@@ -1,30 +1,57 @@
 # SSL Configuration
 
-If you are using TCP then there is the option of adding an SSL certificate to the options hash on initialize.
+The TCP output can connect to Logstash over TLS. The server certificate is always
+verified, and so is the hostname, unless you opt out explicitly.
+
+Enable TLS and verify the server against the system trust store:
+
+```ruby
+LogStashLogger.new(type: :tcp, host: 'logstash.example.com', port: 5228, ssl_enable: true)
+```
+
+If your Logstash server uses a self-signed certificate or a private CA, pass that
+certificate (or CA bundle) as `ssl_certificate`. It is trusted in addition to the
+system trust store, and TLS is enabled implicitly:
 
 ```ruby
-LogStashLogger.new(type: :tcp, port: 5228, ssl_certificate: "/path/to/certificate.crt")
+LogStashLogger.new(type: :tcp, host: 'logstash.example.com', port: 5228, ssl_certificate: "/path/to/certificate.crt")
 ```
 
-The SSL certificate and key can be generated using
+The file is checked when the logger is created, and an `ArgumentError` is raised
+if it is missing or is not a certificate.
+
+A self-signed certificate and key for the Logstash server can be generated with
 
-    openssl req -x509 -batch -nodes -newkey rsa:2048 -keyout logstash.key -out logstash.crt
+    openssl req -x509 -batch -nodes -newkey rsa:2048 -subj "/CN=logstash.example.com" \
+      -addext "subjectAltName=DNS:logstash.example.com" -keyout logstash.key -out logstash.crt
 
-You can also enable SSL without a certificate:
+Use the name your clients connect to in the certificate.
+
+Specify an SSL context for full control over the connection. For example, to
+trust a private CA:
 
 ```ruby
-LogStashLogger.new(type: :tcp, port: 5228, ssl_enable: true)
+ctx = OpenSSL::SSL::SSLContext.new
+ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_PEER, ca_file: '/path/to/ca.pem')
+LogStashLogger.new(type: :tcp, host: 'logstash.example.com', port: 5228, ssl_context: ctx)
 ```
 
-Specify an SSL context to have more control over the behavior. For example,
-set the verify mode:
+### Turning verification off
+
+Verification can only be turned off with an explicit context. This exposes the
+log stream to anyone who can intercept the connection, so use it only for local
+testing:
 
 ```ruby
 ctx = OpenSSL::SSL::SSLContext.new
-ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_NONE)
-LogStashLogger.new(type: :tcp, port: 5228, ssl_context: ctx)
+ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+LogStashLogger.new(type: :tcp, host: 'localhost', port: 5228, ssl_context: ctx, verify_hostname: false)
 ```
 
+Before 1.1.0, `ssl_enable: true` without an `ssl_context` did not verify the
+server certificate. If you relied on that, either pass `ssl_certificate` with the
+server's certificate, or use the explicit context above.
+
 ## Logstash Configuration for SSL
 
 The following Logstash configuration is required for SSL:
@@ -44,47 +71,32 @@ input {
 
 ## Hostname Verification
 
-Hostname verification is enabled by default. Without further configuration,
-the hostname supplied to `:host` will be used to verify the server's certificate
-identity.
-
-If you don't pass an `:ssl_context` or pass a falsey value to the
-`:verify_hostname` option, hostname verification will not occur.
+Hostname verification is enabled by default whenever TLS is used. The hostname
+supplied to `:host` is checked against the server's certificate, and it is also
+sent as the TLS server name (SNI) unless it is an IP address.
 
-### Examples
+**Verify a hostname different from the `:host` option**
 
-**Verify the hostname from the `:host` option**
+This is useful when connecting by IP address:
 
 ```ruby
-ctx = OpenSSL::SSL::SSLContext.new
-ctx.cert = '/path/to/cert.pem'
-ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
-
 LogStashLogger.new \
   type: :tcp,
-  host: 'logstash.example.com'
+  host: '1.2.3.4',
   port: 5228,
-  ssl_context: ctx
+  ssl_certificate: '/path/to/certificate.crt',
+  verify_hostname: 'logstash.example.com'
 ```
 
-**Verify a hostname different from the `:host` option**
-
-```ruby
-LogStashLogger.new \
-  type: :tcp,
-  host: '1.2.3.4'
-  port: 5228,
-  ssl_context: ctx,
-  verify_hostname: 'server.example.com'
-```
+**Disable hostname verification**
 
-**Explicitly disable hostname verification**
+The certificate chain is still verified:
 
 ```ruby
 LogStashLogger.new \
   type: :tcp,
-  host: '1.2.3.4'
+  host: '1.2.3.4',
   port: 5228,
-  ssl_context: ctx,
+  ssl_certificate: '/path/to/certificate.crt',
   verify_hostname: false
 ```

data/docs/troubleshooting.md

@@ -15,6 +15,24 @@ Your application is probably attempting to log data that is not encoded in a val
 standard JSON library will raise an exception. You may be able to overcome this by swapping out a different JSON encoder
 such as Oj. Use the [oj_mimic_json](https://github.com/ohler55/oj_mimic_json) gem to use Oj for JSON generation.
 
+## Records tagged `_logstashlogger_format_failure`
+
+When a log call cannot be turned into a record, for example because a logged
+value cannot be serialized to JSON, LogStashLogger writes a placeholder record
+instead. It is framed like any other record, so the records around it are not
+affected. The placeholder has the message `Failed to format log event`, the
+tag `_logstashlogger_format_failure`, the logger's time and severity, and the
+error class in an `error` field. The error itself is reported to the error
+logger. Before 1.1.0 a bare `Failed to format log event` string was written
+without a line ending, which could merge with the next record.
+
+## The `cee_syslog` header shows the local hostname
+
+The `cee_syslog` formatter writes a `hostname progname:@cee:` header before
+each record. The hostname is always this machine's hostname, and control
+characters in the header are replaced with spaces. A `host` field in the logged
+content, or one set in `customize_event`, appears only in the JSON body.
+
 ## No logs getting sent on Heroku
 
 Heroku recommends installing the [rails_12factor](https://github.com/heroku/rails_12factor) so that logs get sent to STDOUT.
@@ -50,6 +68,10 @@ for other fields.
 
 # Breaking changes
 
+## Version 1.1+
+
+MRI Ruby < 3.3 and Rails < 8.0 are no longer supported, since they have been EOL'ed. If you are on Ruby 3.2 or Rails 7.2, you will need to use 1.0.x.
+
 ## Version 0.27+
 
 MRI Ruby < 3.2 is no longer supported, since it has been EOL'ed. If you are on an older version of Ruby, you will need to use 0.26 or below.

data/gemfiles/rails_8.0.gemfile

@@ -2,7 +2,9 @@
 
 source "https://rubygems.org"
 
-gem "codeclimate-test-reporter", group: :test, require: nil
+gem "simplecov", ">= 0.22", group: :test, require: false
+gem "simplecov-cobertura", group: :test, require: false
 gem "rails", "~> 8.0.0"
+gem "json", "< 3"
 
 gemspec path: "../"

data/gemfiles/rails_8.1.gemfile

@@ -2,7 +2,8 @@
 
 source "https://rubygems.org"
 
-gem "codeclimate-test-reporter", group: :test, require: nil
+gem "simplecov", ">= 0.22", group: :test, require: false
+gem "simplecov-cobertura", group: :test, require: false
 gem "rails", "~> 8.1.0"
 
 gemspec path: "../"

data/lib/logstash-logger/buffer.rb

@@ -180,6 +180,16 @@ module LogStashLogger
       end
     end
 
+    # Put items back into the pending queue for the next flush, without
+    # applying the full-buffer policy. Safe to call from inside a flush.
+    def buffer_requeue(items, group=nil)
+      return if items.empty?
+      @buffer_state[:pending_mutex].synchronize do
+        @buffer_state[:pending_items][group].concat(items)
+        @buffer_state[:pending_count] += items.size
+      end
+    end
+
     # Try to flush events.
     #
     # Returns immediately if flushing is not necessary/possible at the moment:
@@ -305,11 +315,12 @@ module LogStashLogger
         end
     end
 
+    # Never kills the timer from inside itself: called on the timer thread,
+    # this keeps the running timer, and reset_buffer reuses it.
     def reset_flush_timer_thread
-      unless @flush_timer_thread.nil?
-        @flush_timer_thread.kill
-        @flush_timer_thread = nil
-      end
+      return if @flush_timer_thread.nil? || @flush_timer_thread == Thread.current
+      @flush_timer_thread.kill
+      @flush_timer_thread = nil
     end
 
     def buffer_clear_pending

data/lib/logstash-logger/device/aws_stream.rb

@@ -18,6 +18,7 @@ module LogStashLogger
       end
 
       attr_accessor :aws_region, :stream
+      attr_reader :max_record_bytes
 
       def initialize(opts)
         super
@@ -25,6 +26,22 @@ module LogStashLogger
         @secret_access_key = opts[:aws_secret_access_key]
         @aws_region = opts[:aws_region]
         @stream = opts[:stream] || DEFAULT_STREAM
+        # The largest record the stream accepts. Kinesis streams can be
+        # configured above the 1 MiB default, up to 10 MiB.
+        @max_record_bytes = Integer(opts[:aws_max_record_bytes] || self.class.max_record_bytes)
+      end
+
+      # Service limits, defined by each subclass
+      def self.max_records
+        fail NotImplementedError
+      end
+
+      def self.max_record_bytes
+        fail NotImplementedError
+      end
+
+      def self.max_request_bytes
+        fail NotImplementedError
       end
 
       def transform_message(message)
@@ -55,27 +72,32 @@ module LogStashLogger
         yield
       rescue => e
         log_error(e)
-        log_warning("giving up")
         close(flush: false)
+        raise
       end
 
       def write_batch(messages, _group = nil)
-        records = messages.map{ |m| transform_message(m) }
-
-        with_connection do
-          resp = put_records(records)
-
-          # Put any failed records back into the buffer
-          if !is_successful_response(resp)
-            get_response_records(resp).each_with_index do |record, index|
-              if self.class.recoverable_error_codes.include?(record.error_code)
-                log_warning("Failed to post record using #{self.class.stream_class.name} with error: #{record.error_code} #{record.error_message}")
-                log_warning("Retrying")
-                write(records[index][:data])
-              elsif !record.error_code.nil? && record.error_code != ''
-                log_error("Failed to post record using #{self.class.stream_class.name} with error: #{record.error_code} #{record.error_message}")
-              end
-            end
+        records = []
+        messages.each do |message|
+          record = transform_message(message.to_s)
+          size = record_size(record)
+          if size > max_record_bytes
+            log_warning("dropping a record of #{size} bytes: over the #{max_record_bytes}-byte limit of #{self.class.stream_class.name}")
+          else
+            records << record
+          end
+        end
+
+        slices = request_slices(records)
+        slices.each_with_index do |slice, index|
+          begin
+            with_connection { send_slice(slice) }
+          rescue StandardError
+            # Already logged by with_connection. Only records that were not
+            # sent go back; dropped oversized records never do.
+            unsent = slices[index..-1].flatten(1).map { |r| r[:data] }
+            give_back(unsent)
+            return
           end
         end
       end
@@ -88,6 +110,63 @@ module LogStashLogger
         @io = nil
       end
 
+      private
+
+      def send_slice(records)
+        resp = put_records(records)
+        return if is_successful_response(resp)
+
+        retries = []
+        retry_codes = []
+        get_response_records(resp).each_with_index do |record, index|
+          if self.class.recoverable_error_codes.include?(record.error_code)
+            retries << records[index][:data]
+            retry_codes << record.error_code
+          elsif !record.error_code.nil? && record.error_code != ''
+            log_warning("dropping a record that #{self.class.stream_class.name} rejected: #{record.error_code} #{record.error_message}")
+          end
+        end
+        return if retries.empty?
+
+        log_warning("#{retries.size} records will be retried on the next flush (#{retry_codes.tally.map { |c, n| "#{c}: #{n}" }.join(', ')})")
+        # This runs inside a flush, so requeue without the full-buffer check,
+        # which would count the batch being flushed against the limit.
+        buffer_requeue(retries, @buffer_group)
+      end
+
+      def record_size(record)
+        record[:data].bytesize + record.fetch(:partition_key, '').bytesize
+      end
+
+      # Groups records into requests that fit the service's record-count and
+      # request-size limits.
+      def request_slices(records)
+        slices = []
+        bytes = 0
+        records.each do |record|
+          size = record_size(record)
+          if slices.empty? || slices.last.size >= self.class.max_records || bytes + size > self.class.max_request_bytes
+            slices << []
+            bytes = 0
+          end
+          slices.last << record
+          bytes += size
+        end
+        slices
+      end
+
+      # Records from a request that failed: retried on the next flush, or
+      # dropped when drop_messages_on_flush_error is set.
+      def give_back(messages)
+        return if messages.empty?
+        if @drop_messages_on_flush_error
+          log_warning("dropping #{messages.size} records after a failed request")
+        else
+          log_warning("#{messages.size} records will be retried on the next flush")
+          buffer_requeue(messages, @buffer_group)
+        end
+      end
+
     end
   end
 end

data/lib/logstash-logger/device/connectable.rb

@@ -36,6 +36,11 @@ module LogStashLogger
 
         @buffer_logger = opts[:buffer_logger]
 
+        # Optional cap on the size of a single formatted record. Off by default.
+        # Deleted from opts so outputs that forward their options (Redis) never
+        # receive it.
+        @max_record_bytes = parse_max_record_bytes(opts.delete(:max_record_bytes))
+
         buffer_initialize(
           max_items: @buffer_max_items,
           max_interval: @buffer_max_interval,
@@ -48,7 +53,15 @@ module LogStashLogger
       end
 
       def write(message)
-        buffer_receive(message, @buffer_group) unless message.nil?
+        return if message.nil?
+        if @max_record_bytes
+          size = message.to_s.bytesize
+          if size > @max_record_bytes
+            log_warning("dropping a record of #{size} bytes: over max_record_bytes (#{@max_record_bytes})")
+            return
+          end
+        end
+        buffer_receive(message, @buffer_group)
       end
 
       def flush(*args)
@@ -109,6 +122,19 @@ module LogStashLogger
         connect
       end
 
+      private
+
+      def parse_max_record_bytes(value)
+        return nil if value.nil? || value.to_s.strip.empty?
+        bytes = Integer(value)
+        raise ArgumentError, "max_record_bytes must be a positive number of bytes: #{value.inspect}" unless bytes > 0
+        bytes
+      rescue TypeError
+        raise ArgumentError, "max_record_bytes must be a positive number of bytes: #{value.inspect}"
+      end
+
+      public
+
       # Ensure the block is executed with a valid connection
       def with_connection()
         connect unless connected?

data/lib/logstash-logger/device/firehose.rb

@@ -16,6 +16,12 @@ module LogStashLogger
         "ServiceUnavailableException"
       ].freeze
 
+      # https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecordBatch.html
+      # 1,000 KiB per record and 4 MB per request; these cannot be changed.
+      def self.max_records; 500; end
+      def self.max_record_bytes; 1_024_000; end
+      def self.max_request_bytes; 4_000_000; end
+
       def transform_message(message)
         {
           data: message