fluent-plugin-s3 1.8.5 → 1.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ad2143d493bcb5b5805df10225cdb068bffc8ae24a1a6838bde343b18a845ed
-  data.tar.gz: faf67f2a0e65e73b385944113a485f509365b3b9f1e8059ef1b06df8c6719607
+  metadata.gz: d21a6392c01d12f1eb7d42afcb9a068157020c0627772895dedad47f1807c788
+  data.tar.gz: 22cef56213063ea0493c2b1599d2ff8019ab1f0da44e1c6598497100b695859f
 SHA512:
-  metadata.gz: 51c6cd59240d0d2055260b779e9887ab5a8fc0839246cf9154566271de72e30a73ba0587ae2945dd275c1b16b71c43e165a77e43419e73279fbb938904790a48
-  data.tar.gz: ff8aa7a1c32cf273527165eee2dec5df4df217d20e5047c53908469bf13fd8baf6b803781e30d09126d83e0ba4658bf8b28ae2b034e403f627ae9a9e98b312a3
+  metadata.gz: 9e516fbe5805ae366cbd0c588a64b45d8b4b80d0b4afa65e0ebc8eea2e59c1ea6f50d119493b2516f5db94db0b9d53b83a72022badf3179c624269a1493dafa9
+  data.tar.gz: 7b3f6921d9a6f88eb802b93d8613c0f706abb1162be56e833fe4e65b3084bc7a2c7c10296a05467127e6451fdb10ea66457bc5f4eb2bfe62fe965935d2822649

data/.github/workflows/linux.yml

@@ -25,8 +25,8 @@ jobs:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
-    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/.github/workflows/stale-actions.yml

@@ -10,7 +10,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-    - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
+    - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 30

data/ChangeLog

@@ -1,3 +1,11 @@
+Release 1.8.6 - 2026/09/04
+
+  * in_s3: stop retrying SQS messages that can never succeed (GitHub: #484)
+  * in_s3: allow duplicate keys in JSON parser (GitHub: #481)
+  * in_s3/out_s3: mask `proxy_uri` in the configuration dump (GitHub: #486)
+  * out_s3: build the chunk debug message only when debug is on (GitHub: #485)
+  * out_s3: fix parameter descriptions of `checksum_algorithm` and `s3_metadata` (GitHub: #487)
+
 Release 1.8.5 - 2026/06/25
 
   * in_s3: enforce size limits on decompressed payloads

data/Gemfile

@@ -1,3 +1,3 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
 gemspec

data/Gemfile.v0.12

@@ -1,4 +1,4 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
 gem 'json', '= 1.8.3'
 gem 'fluentd', '~> 0.12.0'

data/VERSION

@@ -1 +1 @@
-1.8.5
+1.8.6

data/docs/input.md

@@ -33,6 +33,7 @@ Whether or not object metadata should be added to the record. Defaults to `false
 
 The size limit of the decompressed data. The default is `256m` (256 MiB).
 This parameter is designed to prevent memory exhaustion when extracting highly compressed objects from S3.
+An object over the limit is never ingested and its SQS message is deleted, so raising the limit afterwards will not bring the notification back. See [Handling of failed messages](#handling-of-failed-messages).
 
 ## match_regexp
 
@@ -75,7 +76,7 @@ Check AWS key on start. Default is true.
 
 ## proxy_uri
 
-URI of proxy environment.
+URI of proxy environment. The value is masked in the configuration dump, so credentials embedded in the URI are not written to the log.
 
 ## \<sqs\> section
 
@@ -109,4 +110,12 @@ Interval to retry polling SQS if polling unsuccessful, in seconds. Default is 30
 
 ### event_bridge_mode
 When true, Amazon S3 Event Notification should be configured using the EventBridge integration. Default is false.
-See [Configure S3 event notification using EventBridge](https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html) for additional information.
+See [Configure S3 event notification using EventBridge](https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html) for additional information.
+
+# Handling of failed messages
+
+When handling an SQS message fails, the plugin either deletes the message or leaves it in the queue for redelivery.
+
+A failure that cannot change on redelivery is reported with a warning and the message is deleted. This covers a body that is not an S3 notification, an object key that S3 would reject, an object that the configured `store_as` cannot extract, an object over `decompression_size_limit`, and a line that the configured parser refuses. The object in S3 is not touched, but it is never ingested and the notification is gone, so a mistake in `store_as`, `<parse>` or `decompression_size_limit` loses the notifications that arrive while it is in effect. The warning names the object key, truncated to 256 bytes, so that such an object can be ingested by hand once the configuration is corrected.
+
+Every other failure, such as a networking error, a throttle, a missing object or a bug in the plugin, leaves the message in the queue and SQS redelivers it after the visibility timeout. Configure a [redrive policy with a dead letter queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) so that such a message is moved aside after a bounded number of attempts instead of being retried until the retention period expires.

data/docs/output.md

@@ -312,7 +312,7 @@ Check AWS key on start. Default is true.
 
 ## proxy_uri
 
-uri of proxy environment.
+uri of proxy environment. The value is masked in the configuration dump, so credentials embedded in the URI are not written to the log.
 
 ## path
 

data/lib/fluent/plugin/in_s3.rb

@@ -5,6 +5,8 @@ require 'aws-sdk-s3'
 require 'aws-sdk-sqs'
 require 'aws-sdk-sqs/queue_poller'
 require 'cgi/util'
+require 'json'
+require 'msgpack'
 require 'zlib'
 require 'time'
 require 'tempfile'
@@ -22,7 +24,17 @@ module Fluent::Plugin
     end
 
     DEFAULT_PARSE_TYPE = "none"
+    UNRECOVERABLE_ERRORS = [
+      Fluent::UnrecoverableError,
+      TypeError,
+      ArgumentError,
+      EncodingError,
+      JSON::ParserError,
+      MessagePack::UnpackError,
+      Zlib::Error,
+    ].freeze
     DECOMPRESSION_SIZE_LIMIT = 256 * 1024 * 1024
+    LOG_MAX_BYTES = 256
 
     desc "Use aws-sdk-ruby bundled cert"
     config_param :use_bundled_cert, :bool, default: false
@@ -97,7 +109,7 @@ module Fluent::Plugin
     desc "Check AWS key on start"
     config_param :check_apikey_on_start, :bool, default: true
     desc "URI of proxy environment"
-    config_param :proxy_uri, :string, default: nil
+    config_param :proxy_uri, :string, default: nil, secret: true
     desc "Optional RegEx to match incoming messages"
     config_param :match_regexp, :regexp, default: nil
 
@@ -213,17 +225,18 @@ module Fluent::Plugin
       begin
         @poller.poll(options) do |message|
           begin
-            body = JSON.parse(message.body)
-            log.debug(body)
-            next unless is_valid_queue(body) # skip test queue
-            if @match_regexp
-              raw_key = get_raw_key(body)
-              key = CGI.unescape(raw_key)
-              next unless @match_regexp.match?(key)
-            end
-            process(body)
+            raw_key, key = object_key_from(message)
+            next if raw_key.nil?
+
+            process(raw_key, key)
+          rescue *UNRECOVERABLE_ERRORS => e
+            attributes = { message_id: message.message_id, error_class: e.class.to_s, error: e.message }
+            attributes[:key] = loggable(raw_key) if raw_key
+            log.warn("Skipping SQS message that cannot be handled", attributes)
+            log.warn_backtrace(e.backtrace)
+            next
           rescue => e
-            log.warn(error: e)
+            log.warn("Failed to handle SQS message", message_id: message.message_id, error: e)
             log.warn_backtrace(e.backtrace)
             throw :skip_delete
           end
@@ -235,6 +248,36 @@ module Fluent::Plugin
       end
     end
 
+    def loggable(value)
+      value.byteslice(0, LOG_MAX_BYTES).scrub("?")
+    end
+
+    def object_key_from(message)
+      log.debug { "SQS message #{message.message_id}: #{message.body.dump}" }
+
+      body = JSON.parse(message.body, allow_duplicate_key: true)
+      unless body.is_a?(Hash)
+        log.warn("Skipping SQS message whose body is not a JSON object", message_id: message.message_id)
+        return nil
+      end
+
+      unless is_valid_queue(body)
+        log.warn("Skipping SQS message that is not an S3 notification", message_id: message.message_id)
+        return nil
+      end
+
+      raw_key = get_raw_key(body)
+      if raw_key.nil?
+        log.warn("Skipping SQS message with a malformed notification body", message_id: message.message_id)
+        return nil
+      end
+
+      key = CGI.unescape(raw_key)
+      return nil if @match_regexp && !@match_regexp.match?(key)
+
+      [raw_key, key]
+    end
+
     def is_valid_queue(body)
       if @sqs.event_bridge_mode
         log.debug("checking for eventbridge property")
@@ -247,9 +290,9 @@ module Fluent::Plugin
 
     def get_raw_key(body)
       if @sqs.event_bridge_mode
-        body["detail"]["object"]["key"]
+        body.dig("detail", "object", "key")
       else
-        body["Records"].first["s3"]["object"]["key"]
+        body.dig("Records", 0, "s3", "object", "key")
       end
     end
 
@@ -348,10 +391,7 @@ module Fluent::Plugin
       raise "can't call S3 API. Please check your credentials or s3_region configuration. error = #{e.inspect}"
     end
 
-    def process(body)
-      raw_key = get_raw_key(body)
-      key = CGI.unescape(raw_key)
-
+    def process(raw_key, key)
       io = @bucket.object(key).get.body
       content = @extractor.extract(io)
       es = Fluent::MultiEventStream.new
@@ -368,7 +408,8 @@ module Fluent::Plugin
     end
 
     class Extractor
-      class SizeLimitError < StandardError; end
+      class SizeLimitError < Fluent::UnrecoverableError; end
+      class CommandError < Fluent::UnrecoverableError; end
 
       include Fluent::Configurable
 
@@ -431,10 +472,16 @@ module Fluent::Plugin
               end
             end
 
-            if wait_thr.value.success?
+            status = wait_thr.value
+            if status.success?
               out
+            elsif status.signaled?
+              # not a CommandError: a signal comes from outside the data, so a retry may succeed
+              raise "Command #{command} was terminated by signal #{status.termsig}"
             else
-              raise "Command execution failed: #{command} (status: #{wait_thr.value})"
+              stderr_tail = stderr.read.strip[0, 200]
+              reason = stderr_tail.empty? ? "" : ": #{stderr_tail}"
+              raise CommandError, "Command #{command} exited with #{status.exitstatus}#{reason}"
             end
           end
         end

data/lib/fluent/plugin/out_s3.rb

@@ -121,7 +121,7 @@ module Fluent::Plugin
     desc "Check AWS key on start"
     config_param :check_apikey_on_start, :bool, default: true
     desc "URI of proxy environment"
-    config_param :proxy_uri, :string, default: nil
+    config_param :proxy_uri, :string, default: nil, secret: true
     desc "Use S3 reduced redundancy storage for 33% cheaper pricing. Deprecated. Use storage_class instead"
     config_param :reduced_redundancy, :bool, default: false, deprecated: "Use storage_class parameter instead."
     desc "The type of storage to use for the object(STANDARD,REDUCED_REDUNDANCY,STANDARD_IA)"
@@ -162,9 +162,9 @@ module Fluent::Plugin
     config_param :warn_for_delay, :time, default: nil
     desc "Arbitrary S3 tag-set for the object"
     config_param :tagging, :string, default: nil
-    desc "Arbitrary S3 metadata headers to set for the object"
-    config_param :checksum_algorithm, :string, default: nil
     desc "Indicates the algorithm you want Amazon S3 to use to create the checksum for the object (CRC32,CRC32C,SHA1,SHA256)"
+    config_param :checksum_algorithm, :string, default: nil
+    desc "Arbitrary S3 metadata headers to set for the object"
     config_param :s3_metadata, :hash, default: nil
     config_section :bucket_lifecycle_rule, param_name: :bucket_lifecycle_rules, multi: true do
       desc "A unique ID for this rule"
@@ -362,7 +362,7 @@ module Fluent::Plugin
       begin
         @compressor.compress(chunk, tmp)
         tmp.rewind
-        log.debug "out_s3: write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}"
+        log.debug { "out_s3: write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}" }
 
         put_options = {
           body: tmp,
@@ -390,7 +390,7 @@ module Fluent::Plugin
         end
         @bucket.object(s3path).put(put_options)
 
-        log.debug "out_s3: completed to write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}"
+        log.debug { "out_s3: completed to write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}" }
 
         @values_for_s3_object_chunk.delete(chunk.unique_id)
 

data/lib/fluent/plugin/s3_extractor_gzip_command.rb

@@ -21,7 +21,7 @@ module Fluent::Plugin
       def extract(io)
         begin
           extract_with_command("gzip #{@command_parameter}", io, "gzip-temp")
-        rescue SizeLimitError
+        rescue Fluent::UnrecoverableError
           raise
         rescue => e
           log.warn "gzip command execution failed: #{e.message}. Fallback to GzipExtractor."

data/lib/fluent/plugin/s3_extractor_lzma2.rb

@@ -21,10 +21,10 @@ module Fluent::Plugin
       def extract(io)
         begin
           extract_with_command("xz #{@command_parameter}", io, "xz-temp")
-        rescue SizeLimitError
+        rescue Fluent::UnrecoverableError
           raise
-        rescue
-          raise "Failed to extract #{path} with xz command."
+        rescue => e
+          raise "Failed to extract with the xz command: #{e.message}"
         end
       end
     end

data/lib/fluent/plugin/s3_extractor_lzo.rb

@@ -21,10 +21,10 @@ module Fluent::Plugin
       def extract(io)
         begin
           extract_with_command("lzop #{@command_parameter}", io, "lzop-temp")
-        rescue SizeLimitError
+        rescue Fluent::UnrecoverableError
           raise
-        rescue
-          raise "Failed to extract #{path} with lzop command."
+        rescue => e
+          raise "Failed to extract with the lzop command: #{e.message}"
         end
       end
     end

data/test/test_in_s3.rb

@@ -834,4 +834,281 @@ EOS
     assert_equal({ "message" => "aaa" }, events.first[2])
   end
 
+  POLL_CONFIG = CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\n"
+
+  EVENT_BRIDGE_POLL_CONFIG =
+    CONFIG.sub("<sqs>", "<sqs>\n      event_bridge_mode true") +
+    "\ncheck_apikey_on_start false\nstore_as text\nformat none\n"
+
+  def notification(key)
+    JSON.generate({ "Records" => [{ "s3" => { "object" => { "key" => key } } }] })
+  end
+
+  def gzipped(text)
+    buffer = StringIO.new(String.new, "wb")
+    Zlib::GzipWriter.wrap(buffer) { |gz| gz.write(text) }
+    buffer.string
+  end
+
+  def stub_s3_object(payload: "aaa", get_error: nil)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new(payload) }
+    s3_object = stub(Object.new)
+    s3_object.get { raise get_error if get_error; s3_response }
+    @s3_bucket.object(anything).at_least(0) do |key|
+      raise ArgumentError, ":key must not be blank" if key.to_s.empty?
+      raise ArgumentError, "invalid byte sequence in UTF-8" unless key.to_s.valid_encoding?
+      s3_object
+    end
+  end
+
+  def poll_once(d, message)
+    deleted = []
+    @sqs_poller.delete_messages(anything) { |messages| deleted.concat(messages) }
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    deleted
+  end
+
+  def warn_logs(d)
+    d.logs.select { |log| log.include?("[warn]") }
+  end
+
+  data(
+    "body is not JSON" => "hello",
+    "body is empty" => "",
+    "body is not a JSON object" => "[]",
+    "body is null" => "null",
+    "Records is empty" => '{"Records":[]}',
+    "record without s3" => '{"Records":[{}]}',
+    "record without object" => '{"Records":[{"s3":{}}]}',
+    "record without key" => '{"Records":[{"s3":{"object":{}}}]}',
+    "Records is a string" => '{"Records":"test_key"}',
+    "Records is an object" => '{"Records":{"s3":{}}}',
+    "key is not a string" => '{"Records":[{"s3":{"object":{"key":1}}}]}',
+    "key is empty" => '{"Records":[{"s3":{"object":{"key":""}}}]}',
+    "key holds invalid utf-8" => %Q({"Records":[{"s3":{"object":{"key":"a\xFF\xFEb"}}}]}),
+    "key decodes to invalid utf-8" => '{"Records":[{"s3":{"object":{"key":"%FF"}}}]}',
+  )
+  def test_unrecoverable_message_is_deleted(body)
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, body)
+    deleted = poll_once(d, message)
+
+    assert_nothing_raised { d.run {} }
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Skipping SQS message") })
+  end
+
+  def test_unrecoverable_message_in_event_bridge_mode_is_deleted
+    setup_mocks
+    stub_s3_object
+    d = create_driver(EVENT_BRIDGE_POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, '{"detail":{}}')
+    deleted = poll_once(d, message)
+
+    assert_nothing_raised { d.run {} }
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Skipping SQS message") })
+  end
+
+  data(
+    "no Records property" => "{}",
+    "sns envelope" => '{"Type":"Notification","Message":"{}"}',
+    "s3 test event" => '{"Service":"Amazon S3","Event":"s3:TestEvent"}',
+  )
+  def test_message_that_is_not_an_s3_notification_is_reported(body)
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+    mock(d.instance).process(anything).never
+
+    message = Struct::StubMessage.new(1, 1, body)
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("not an S3 notification") })
+  end
+
+  data(
+    "missing object" => "NoSuchKey",
+    "missing bucket" => "NoSuchBucket",
+    "access denied" => "AccessDenied",
+    "throttled" => "SlowDown",
+  )
+  def test_service_error_keeps_the_message_for_redelivery(error_name)
+    setup_mocks
+    stub_s3_object(get_error: Aws::S3::Errors.const_get(error_name).new(nil, error_name))
+    d = create_driver(POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Failed to handle SQS message") })
+  end
+
+  def test_networking_error_keeps_the_message_for_redelivery
+    setup_mocks
+    stub_s3_object(get_error: Seahorse::Client::NetworkingError.new(Errno::ECONNRESET.new))
+    d = create_driver(POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([], deleted)
+  end
+
+  data(
+    "gzip" => "gzip",
+    "gzip_command" => "gzip_command",
+    "lzo" => "lzo",
+    "lzma2" => "lzma2",
+  )
+  def test_corrupt_archive_is_deleted(store_type)
+    setup_mocks
+    stub_s3_object(payload: "this is not an archive")
+    begin
+      d = create_driver(POLL_CONFIG.sub("store_as text", "store_as #{store_type}"))
+    rescue Fluent::ConfigError => e
+      pend(e.message)
+    end
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Skipping SQS message") })
+  end
+
+  def test_object_over_the_decompression_limit_is_deleted
+    setup_mocks
+    stub_s3_object(payload: gzipped("a" * 100))
+    d = create_driver(POLL_CONFIG.sub("store_as text", "store_as gzip") + "\ndecompression_size_limit 10\n")
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(d.logs.any? { |log| log.include?("Extracted data exceeds limit") })
+  end
+
+  def test_unrecoverable_message_is_not_redelivered
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+
+    deleted = []
+    attempts = 0
+    @sqs_poller.delete_messages(anything) { |messages| deleted.concat(messages) }
+    message = Struct::StubMessage.new("mid-1", "rh-1", '{"Records":[]}')
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 5
+        d.instance.instance_variable_set(:@running, false)
+      end
+      next [] unless deleted.empty?
+      attempts += 1
+      [message]
+    end
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_equal(1, attempts)
+  end
+
+  def test_a_bug_in_the_plugin_keeps_the_message_for_redelivery
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+    stub(d.instance).is_valid_queue(anything) { raise NoMethodError, "undefined method 'foo' for nil" }
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Failed to handle SQS message") })
+  end
+
+  def test_unparsable_object_is_deleted
+    setup_mocks
+    stub_s3_object(payload: "\xC1")
+    d = create_driver(POLL_CONFIG.sub("format none", "<parse>\n  @type msgpack\n</parse>"))
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Skipping SQS message") })
+  end
+
+  def test_deleting_warn_names_the_object_key
+    setup_mocks
+    stub_s3_object(payload: "this is not an archive")
+    d = create_driver(POLL_CONFIG.sub("store_as text", "store_as gzip"))
+
+    message = Struct::StubMessage.new(1, 1, notification("path/to/broken.gz"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?(%Q(key="path/to/broken.gz")) })
+    assert_true(warn_logs(d).any? { |log| log.include?("in_s3.rb:") })
+  end
+
+  def test_deleting_warn_truncates_a_long_object_key
+    setup_mocks
+    stub_s3_object(payload: "this is not an archive")
+    d = create_driver(POLL_CONFIG.sub("store_as text", "store_as gzip"))
+
+    message = Struct::StubMessage.new(1, 1, notification("\u3042" * 400))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    line = warn_logs(d).find { |log| log.include?("Skipping SQS message") }
+    assert_operator(line[/key="(.*)"/, 1].length, :<=, Fluent::Plugin::S3Input::LOG_MAX_BYTES)
+  end
+
+  def test_a_truncated_object_key_is_still_valid_utf8
+    d = create_driver(POLL_CONFIG)
+    logged = d.instance.__send__(:loggable, "\u3042" * 400)
+
+    assert_true(logged.valid_encoding?)
+    assert_operator(logged.bytesize, :<=, Fluent::Plugin::S3Input::LOG_MAX_BYTES)
+    assert_nothing_raised { JSON.generate([logged]) }
+  end
+
+  def test_valid_message_is_processed_and_deleted
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run(expect_emits: 1)
+
+    assert_equal([message], deleted)
+    assert_equal({ "message" => "aaa" }, d.events.first[2])
+    assert_equal([], warn_logs(d))
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.8.5
+  version: 1.8.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -214,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector
 test_files: