sqs_extended_client 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f4fa6e3026a96530fc60de516f4817f5d2ff08f106f48fef058728e83aafa0b6
+  data.tar.gz: 192fcc74a733f1780e0d50fa6092ad29e462010128c897b8111f27135b675b87
+SHA512:
+  metadata.gz: adbb8b60ba77329fb74f8a52d69397ac41dff4263cb5a932bcca4f158041d766987480aad947a6b15ac91f2e8d6a2deeb660025030cea82fc70cfa6f19cf8e65
+  data.tar.gz: a5d05a27eee220902bec964d8645f1f77c4a9bdea6da9461b676451d5cb42618bd52549027f502dbb764e79aebb0425c106ef47d003d232ebc5ccd4203952a83

data/CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+## [Unreleased]
+
+### Added
+
+- Initial release: `SqsExtendedClient::Client`, an `Aws::SQS::Client` wrapper that offloads
+  message bodies larger than 256 KB to S3 and restores them on receive.

data/LICENSE.txt

@@ -0,0 +1,204 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+   Copyright 2026 Ngoc Bui

data/NOTICE

@@ -0,0 +1,20 @@
+sqs_extended_client
+Copyright 2026 Ngoc Bui
+
+This product is an independent Ruby implementation of the payload offloading
+protocol introduced by the Amazon SQS Extended Client Library for Java:
+
+  https://github.com/awslabs/amazon-sqs-java-extended-client-lib
+  Copyright 2010-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+  Licensed under the Apache License, Version 2.0.
+
+The wire format reproduced here -- the ExtendedPayloadSize message attribute,
+the PayloadS3Pointer message body and the receipt handle markers -- originates
+in that library and in its supporting library:
+
+  https://github.com/awslabs/payload-offloading-java-common-lib-for-aws
+  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+  Licensed under the Apache License, Version 2.0.
+
+No source code from those projects is included in this product. This project is
+not affiliated with, endorsed by, or sponsored by Amazon Web Services.

data/README.md

@@ -0,0 +1,182 @@
+# sqs_extended_client
+
+Send and receive Amazon SQS messages larger than the 256 KB limit by storing the body in S3
+and putting a small pointer on the queue instead. It is a wrapper around `Aws::SQS::Client`,
+so anything it does not need to touch is delegated straight through.
+
+The wire format matches the AWS
+[Java](https://github.com/awslabs/amazon-sqs-java-extended-client-lib) and
+[Python](https://github.com/awslabs/amazon-sqs-python-extended-client-lib) extended clients:
+a Ruby producer can publish to a queue a Java consumer drains, and the other way round.
+
+## Installation
+
+```ruby
+gem "sqs_extended_client"
+```
+
+Requires Ruby 3.1 or newer, and is tested on Ruby 4.
+
+## Usage
+
+Build the client once and hold onto it. Pass your own SQS and S3 clients when you need to set
+region, credentials or retries; every other keyword is an option from the table below.
+
+```ruby
+client = SqsExtendedClient::Client.new(
+  bucket_name: "my-payload-bucket",
+  sqs_client: Aws::SQS::Client.new(region: "ap-southeast-1"),
+  s3_client: Aws::S3::Client.new(region: "ap-southeast-1")
+)
+```
+
+### Sending
+
+Send as you would with the plain SQS client. Bodies under the threshold go straight to the
+queue; anything larger is written to S3 first and the queue gets a pointer.
+
+```ruby
+client.send_message(
+  queue_url: queue_url,
+  message_body: JSON.generate(order),  # any size
+  message_attributes: {
+    "event_type" => { data_type: "String", string_value: "order.created" }
+  }
+)
+```
+
+Batches work the same way, each entry is offloaded on its own:
+
+```ruby
+client.send_message_batch(
+  queue_url: queue_url,
+  entries: orders.each_with_index.map do |order, index|
+    { id: index.to_s, message_body: JSON.generate(order) }
+  end
+)
+```
+
+Remember that SQS sizes a batch by what actually goes on the queue. Offloaded entries are only
+a pointer each, so a batch of large payloads still fits comfortably under the 256 KB batch cap.
+
+### Polling
+
+A worker loop. `receive_message` fetches the body back from S3 before you see it, and
+`delete_message` removes the S3 object along with the message:
+
+```ruby
+loop do
+  response = client.receive_message(
+    queue_url: queue_url,
+    max_number_of_messages: 10,
+    wait_time_seconds: 20,   # long polling, keeps the loop from spinning
+    visibility_timeout: 60   # must comfortably exceed your processing time
+  )
+
+  response.messages.each do |message|
+    process(JSON.parse(message.body))
+
+    client.delete_message(queue_url: queue_url, receipt_handle: message.receipt_handle)
+  rescue StandardError => e
+    # Leave the message alone and let SQS redeliver it after the visibility timeout.
+    logger.error("failed to process #{message.message_id}: #{e.message}")
+  end
+end
+```
+
+Delete only after the work succeeded. Deleting first would drop the S3 object too, so a
+redelivery would arrive pointing at an object that is no longer there.
+
+To delete in batches, pass the receipt handles through unchanged — the pointer travels inside
+them, and each entry is cleaned up from S3 individually:
+
+```ruby
+client.delete_message_batch(
+  queue_url: queue_url,
+  entries: processed.each_with_index.map do |message, index|
+    { id: index.to_s, receipt_handle: message.receipt_handle }
+  end
+)
+```
+
+If processing runs long, extend the lease with `change_message_visibility`; it accepts the
+receipt handle you were given and strips the pointer out before the call reaches SQS:
+
+```ruby
+client.change_message_visibility(
+  queue_url: queue_url,
+  receipt_handle: message.receipt_handle,
+  visibility_timeout: 300
+)
+```
+
+## Options
+
+| Option | Default | Meaning |
+| --- | --- | --- |
+| `bucket_name` | required | S3 bucket the payloads are written to. |
+| `payload_size_threshold` | `262_144` | Offload once body + attributes exceed this many bytes. |
+| `always_through_s3` | `false` | Offload every message regardless of size. |
+| `cleanup_s3_payload` | `true` | Delete the S3 object when the message is deleted. |
+| `ignore_payload_not_found` | `false` | On a missing S3 object, delete the SQS message and skip it instead of raising. |
+| `use_legacy_reserved_attribute_name` | `false` | Flag messages with `SQSLargePayloadSize` instead of `ExtendedPayloadSize`. |
+| `s3_key_prefix` | `""` | Prefix for generated keys, e.g. `"payloads/"`. |
+| `s3_put_object_options` | `{}` | Merged into `put_object`, e.g. `{ server_side_encryption: "aws:kms" }`. |
+
+## How it works
+
+An offloaded message carries a `Number` attribute named `ExtendedPayloadSize` holding the
+original body size, and its body is replaced by
+
+```json
+["software.amazon.payloadoffloading.PayloadS3Pointer",{"s3BucketName":"...","s3Key":"..."}]
+```
+
+The first element is the Java class name the original implementation serialized with. This
+gem always writes the current name and ignores whatever it reads, so messages from the legacy
+v1 Java client — which used `com.amazon.sqs.javamessaging.MessageS3Pointer` and the
+`SQSLargePayloadSize` attribute — are read without any extra configuration.
+
+`delete_message` only receives a receipt handle, so on receive the bucket and key are spliced
+into the handle between markers:
+
+```
+-..s3BucketName..-{bucket}-..s3BucketName..--..s3Key..-{key}-..s3Key..-{real handle}
+```
+
+`delete_message`, `delete_message_batch` and `change_message_visibility` strip that back off
+before the call reaches SQS. Because a message received here may be deleted by a Java or
+Python consumer (and vice versa), those markers are part of the wire contract.
+
+Only the operations that need it are overridden: `send_message`, `send_message_batch`,
+`receive_message`, `delete_message`, `delete_message_batch`, `change_message_visibility` and
+`change_message_visibility_batch`. Every other SQS call is forwarded unchanged.
+
+## Caveats
+
+- SQS allows 10 message attributes and the reserved one takes a slot, so an offloaded message
+  may carry at most 9 of your own.
+- Nothing here expires S3 objects on its own. Set a lifecycle rule on the bucket so payloads
+  from messages that were never consumed do not accumulate.
+- The consumer needs `s3:GetObject` and, for cleanup, `s3:DeleteObject` on the bucket.
+- On a FIFO queue with content based deduplication, the hash is taken over the pointer, which
+  is unique per send. Pass an explicit `message_deduplication_id` for offloaded messages.
+- `receive_message` returns a freshly built `Aws::SQS::Types::ReceiveMessageResult`, so the
+  Seahorse response context (`.context`, `#successful?`) is not carried through.
+
+## Credits
+
+The design is not original to this gem. The approach, and every byte of the wire format it
+speaks, come from the [Amazon SQS Extended Client Library for Java](https://github.com/awslabs/amazon-sqs-java-extended-client-lib)
+and its supporting [payload offloading library](https://github.com/awslabs/payload-offloading-java-common-lib-for-aws),
+both Apache-2.0 licensed and copyright Amazon.com, Inc. or its affiliates. AWS also publishes
+an [official Python implementation](https://github.com/awslabs/amazon-sqs-python-extended-client-lib).
+
+This is an independent reimplementation in Ruby. No source code from those projects is
+included here, and the project is not affiliated with, endorsed by, or sponsored by Amazon
+Web Services.
+
+## License
+
+Apache-2.0, matching the library this one follows. See [LICENSE.txt](LICENSE.txt) and
+[NOTICE](NOTICE).

data/lib/sqs_extended_client/client.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module SqsExtendedClient
+  # A drop-in wrapper around Aws::SQS::Client that offloads large message bodies
+  # to S3 and puts them back on the way out, wire compatible with the AWS Java
+  # and Python extended clients. Anything it does not override is delegated
+  # untouched to the underlying SQS client.
+  class Client
+    RESERVED_ATTRIBUTE_NAME = "ExtendedPayloadSize"
+    LEGACY_RESERVED_ATTRIBUTE_NAME = "SQSLargePayloadSize"
+    RESERVED_ATTRIBUTE_NAMES = [RESERVED_ATTRIBUTE_NAME, LEGACY_RESERVED_ATTRIBUTE_NAME].freeze
+
+    attr_reader :sqs_client, :s3_client, :config
+
+    def initialize(bucket_name:, sqs_client: nil, s3_client: nil, **options)
+      @config = Configuration.new(bucket_name: bucket_name, **options)
+      @sqs_client = sqs_client || Aws::SQS::Client.new
+      @s3_client = s3_client || Aws::S3::Client.new
+      @payload_store = PayloadStore.new(s3_client: @s3_client, config: @config)
+    end
+
+    def send_message(params = {})
+      sqs_client.send_message(**offload(params))
+    end
+
+    def send_message_batch(params = {})
+      sqs_client.send_message_batch(**params, entries: params.fetch(:entries).map { |entry| offload(entry) })
+    end
+
+    def receive_message(params = {})
+      params = params.merge(message_attribute_names: requested_attribute_names(params[:message_attribute_names]))
+      response = sqs_client.receive_message(**params)
+      messages = response.messages.filter_map { |message| restore(message, params.fetch(:queue_url)) }
+      Aws::SQS::Types::ReceiveMessageResult.new(messages: messages)
+    end
+
+    def delete_message(params = {})
+      receipt_handle = params.fetch(:receipt_handle)
+      cleanup_payload(receipt_handle)
+      sqs_client.delete_message(**params, receipt_handle: ReceiptHandle.strip(receipt_handle))
+    end
+
+    def delete_message_batch(params = {})
+      entries = params.fetch(:entries).map do |entry|
+        receipt_handle = entry.fetch(:receipt_handle)
+        cleanup_payload(receipt_handle)
+        entry.merge(receipt_handle: ReceiptHandle.strip(receipt_handle))
+      end
+      sqs_client.delete_message_batch(**params, entries: entries)
+    end
+
+    def change_message_visibility(params = {})
+      sqs_client.change_message_visibility(**params, receipt_handle: ReceiptHandle.strip(params.fetch(:receipt_handle)))
+    end
+
+    def change_message_visibility_batch(params = {})
+      entries = params.fetch(:entries).map do |entry|
+        entry.merge(receipt_handle: ReceiptHandle.strip(entry.fetch(:receipt_handle)))
+      end
+      sqs_client.change_message_visibility_batch(**params, entries: entries)
+    end
+
+    def respond_to_missing?(name, include_private = false)
+      sqs_client.respond_to?(name, include_private) || super
+    end
+
+    def method_missing(name, ...)
+      return super unless sqs_client.respond_to?(name)
+
+      sqs_client.public_send(name, ...)
+    end
+
+    private
+
+    # Send path: swap the body for a pointer and flag the message.
+    def offload(params)
+      attributes = params[:message_attributes] || {}
+      validate_attributes!(attributes)
+
+      body = params.fetch(:message_body)
+      return params unless config.always_through_s3? || large?(body, attributes)
+
+      pointer = @payload_store.put(body)
+      params.merge(
+        message_body: pointer.to_json,
+        message_attributes: attributes.merge(
+          reserved_attribute_name => { data_type: "Number", string_value: body.bytesize.to_s }
+        )
+      )
+    end
+
+    # Receive path: pull the body back out of S3 and hide our bookkeeping.
+    # Returns nil for a message that was dropped, see ignore_payload_not_found.
+    def restore(message, queue_url)
+      attributes = message.message_attributes || {}
+      return message unless RESERVED_ATTRIBUTE_NAMES.any? { |name| attributes.key?(name) }
+
+      pointer = PayloadPointer.from_json(message.body)
+      begin
+        body = @payload_store.get(pointer)
+      rescue Aws::S3::Errors::NoSuchKey
+        raise unless config.ignore_payload_not_found?
+
+        sqs_client.delete_message(queue_url: queue_url, receipt_handle: message.receipt_handle)
+        return nil
+      end
+
+      rebuild(
+        message,
+        body: body,
+        message_attributes: attributes.except(*RESERVED_ATTRIBUTE_NAMES),
+        receipt_handle: ReceiptHandle.embed(message.receipt_handle, pointer)
+      )
+    end
+
+    def cleanup_payload(receipt_handle)
+      return unless config.cleanup_s3_payload? && ReceiptHandle.embedded?(receipt_handle)
+
+      @payload_store.delete(ReceiptHandle.pointer(receipt_handle))
+    end
+
+    # Ask for the reserved attributes on top of whatever the caller wanted, so a
+    # receive that names specific attributes still tells us the body is a pointer.
+    def requested_attribute_names(names)
+      ((names || []) - RESERVED_ATTRIBUTE_NAMES) + RESERVED_ATTRIBUTE_NAMES
+    end
+
+    def reserved_attribute_name
+      config.use_legacy_reserved_attribute_name? ? LEGACY_RESERVED_ATTRIBUTE_NAME : RESERVED_ATTRIBUTE_NAME
+    end
+
+    def large?(body, attributes)
+      body.bytesize + attributes_size(attributes) > config.payload_size_threshold
+    end
+
+    def validate_attributes!(attributes)
+      size = attributes_size(attributes)
+      if size > config.payload_size_threshold
+        raise MessageAttributeError,
+              "message attributes total #{size} bytes, over the #{config.payload_size_threshold} byte threshold; " \
+              "put the payload in the message body instead"
+      end
+
+      if attributes.size > Configuration::MAX_ALLOWED_ATTRIBUTES
+        raise MessageAttributeError,
+              "#{attributes.size} message attributes exceeds the maximum of " \
+              "#{Configuration::MAX_ALLOWED_ATTRIBUTES} for large payload messages"
+      end
+
+      reserved = RESERVED_ATTRIBUTE_NAMES.find { |name| attributes.key?(name) }
+      return if reserved.nil?
+
+      raise MessageAttributeError, "message attribute #{reserved} is reserved by the extended client"
+    end
+
+    def attributes_size(attributes)
+      attributes.sum do |name, value|
+        size = name.to_s.bytesize
+        size += attribute_field(value, :data_type).to_s.bytesize
+        size += attribute_field(value, :string_value).to_s.bytesize
+        size + attribute_field(value, :binary_value).to_s.bytesize
+      end
+    end
+
+    # Attribute values arrive as hashes on the way out and as SDK structs on the
+    # way in; read either.
+    def attribute_field(value, name)
+      return value.public_send(name) unless value.is_a?(Hash)
+
+      value[name] || value[name.to_s]
+    end
+
+    # SDK response structs are rebuilt rather than mutated so nested members survive.
+    def rebuild(message, **overrides)
+      members = message.members.to_h { |member| [member, message[member]] }
+      Aws::SQS::Types::Message.new(**members, **overrides)
+    end
+  end
+end

data/lib/sqs_extended_client/configuration.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module SqsExtendedClient
+  class Configuration
+    # SQS caps a message at 256 KiB; anything larger goes to S3.
+    DEFAULT_PAYLOAD_SIZE_THRESHOLD = 262_144
+
+    # SQS allows 10 message attributes and the reserved one takes a slot.
+    MAX_ALLOWED_ATTRIBUTES = 9
+
+    MAX_S3_KEY_LENGTH = 1024
+    UUID_LENGTH = 36
+    MAX_S3_KEY_PREFIX_LENGTH = MAX_S3_KEY_LENGTH - UUID_LENGTH
+    INVALID_S3_KEY_PREFIX_CHARACTERS = %r{[^a-zA-Z0-9./_-]}
+
+    attr_reader :bucket_name, :s3_key_prefix, :payload_size_threshold, :s3_put_object_options, :always_through_s3,
+                :cleanup_s3_payload, :ignore_payload_not_found, :use_legacy_reserved_attribute_name
+
+    alias always_through_s3? always_through_s3
+    alias cleanup_s3_payload? cleanup_s3_payload
+    alias ignore_payload_not_found? ignore_payload_not_found
+    alias use_legacy_reserved_attribute_name? use_legacy_reserved_attribute_name
+
+    def initialize(
+      bucket_name:,
+      s3_key_prefix: "",
+      payload_size_threshold: DEFAULT_PAYLOAD_SIZE_THRESHOLD,
+      always_through_s3: false,
+      cleanup_s3_payload: true,
+      ignore_payload_not_found: false,
+      use_legacy_reserved_attribute_name: false,
+      s3_put_object_options: {}
+    )
+      raise ConfigurationError, "bucket_name is required" if bucket_name.nil? || bucket_name.to_s.empty?
+
+      @bucket_name = bucket_name
+      @s3_key_prefix = validate_key_prefix(s3_key_prefix)
+      @payload_size_threshold = Integer(payload_size_threshold)
+      @always_through_s3 = always_through_s3
+      @cleanup_s3_payload = cleanup_s3_payload
+      @ignore_payload_not_found = ignore_payload_not_found
+      @use_legacy_reserved_attribute_name = use_legacy_reserved_attribute_name
+      @s3_put_object_options = s3_put_object_options
+      freeze
+    end
+
+    private
+
+    def validate_key_prefix(prefix)
+      trimmed = prefix.to_s.strip
+      return trimmed if trimmed.empty?
+
+      if trimmed.length > MAX_S3_KEY_PREFIX_LENGTH
+        raise ConfigurationError, "s3_key_prefix must be at most #{MAX_S3_KEY_PREFIX_LENGTH} characters"
+      end
+      raise ConfigurationError, "s3_key_prefix must not start with '.' or '/'" if trimmed.start_with?(".", "/")
+      raise ConfigurationError, "s3_key_prefix must not contain '..'" if trimmed.include?("..")
+      if INVALID_S3_KEY_PREFIX_CHARACTERS.match?(trimmed)
+        raise ConfigurationError,
+              "s3_key_prefix may only contain letters, digits, '/', '_', '-' and '.'"
+      end
+
+      trimmed
+    end
+  end
+end

data/lib/sqs_extended_client/errors.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SqsExtendedClient
+  class Error < StandardError; end
+
+  # Raised when the client is built with options it cannot work with.
+  class ConfigurationError < Error; end
+
+  # Raised when a message body is flagged as offloaded but is not a valid pointer.
+  class InvalidPointerError < Error; end
+
+  # Raised when outgoing message attributes violate the extended client's limits.
+  class MessageAttributeError < Error; end
+end

data/lib/sqs_extended_client/payload_pointer.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SqsExtendedClient
+  # The reference that takes the place of an offloaded message body.
+  #
+  # On the wire it is a two element JSON array whose first element is the Java
+  # class name the original implementation serialized with. We always write the
+  # current name and ignore whatever we read, which is how messages from the
+  # legacy v1 Java client stay readable.
+  class PayloadPointer
+    CLASS_NAME = "software.amazon.payloadoffloading.PayloadS3Pointer"
+    LEGACY_CLASS_NAME = "com.amazon.sqs.javamessaging.MessageS3Pointer"
+
+    attr_reader :bucket_name, :key
+
+    def initialize(bucket_name, key)
+      @bucket_name = bucket_name
+      @key = key
+    end
+
+    def self.from_json(json)
+      parsed = JSON.parse(json)
+      unless parsed.is_a?(Array) && parsed.size == 2 && parsed[1].is_a?(Hash)
+        raise InvalidPointerError, "expected a [class_name, pointer] pair, got: #{json.inspect}"
+      end
+
+      body = parsed[1]
+      bucket_name = body["s3BucketName"]
+      key = body["s3Key"]
+      if bucket_name.nil? || key.nil?
+        raise InvalidPointerError,
+              "pointer is missing s3BucketName or s3Key: #{json.inspect}"
+      end
+
+      new(bucket_name, key)
+    rescue JSON::ParserError => e
+      raise InvalidPointerError, "message body is not valid JSON: #{e.message}"
+    end
+
+    def to_json(*_args)
+      JSON.generate([CLASS_NAME, { "s3BucketName" => bucket_name, "s3Key" => key }])
+    end
+
+    def ==(other)
+      other.is_a?(PayloadPointer) && other.bucket_name == bucket_name && other.key == key
+    end
+    alias eql? ==
+
+    def hash
+      [bucket_name, key].hash
+    end
+
+    def to_s
+      "s3://#{bucket_name}/#{key}"
+    end
+  end
+end

data/lib/sqs_extended_client/payload_store.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module SqsExtendedClient
+  # Puts, gets and deletes message bodies in S3.
+  class PayloadStore
+    def initialize(s3_client:, config:)
+      @s3_client = s3_client
+      @config = config
+    end
+
+    def put(payload)
+      key = "#{@config.s3_key_prefix}#{SecureRandom.uuid}"
+      @s3_client.put_object(
+        **@config.s3_put_object_options,
+        bucket: @config.bucket_name,
+        key: key,
+        body: payload
+      )
+      PayloadPointer.new(@config.bucket_name, key)
+    end
+
+    def get(pointer)
+      @s3_client.get_object(bucket: pointer.bucket_name, key: pointer.key).body.read
+    end
+
+    def delete(pointer)
+      @s3_client.delete_object(bucket: pointer.bucket_name, key: pointer.key)
+    end
+  end
+end

data/lib/sqs_extended_client/receipt_handle.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SqsExtendedClient
+  # Deleting a message has to delete its S3 object too, but delete_message only
+  # carries a receipt handle. The Java client solves this by prefixing the real
+  # handle with the bucket and key between markers, and stripping them back off
+  # before the call reaches SQS. The markers below are wire format: a message
+  # received here may well be deleted by a Java or Python consumer.
+  module ReceiptHandle
+    BUCKET_MARKER = "-..s3BucketName..-"
+    KEY_MARKER = "-..s3Key..-"
+
+    module_function
+
+    def embed(receipt_handle, pointer)
+      "#{BUCKET_MARKER}#{pointer.bucket_name}#{BUCKET_MARKER}" \
+        "#{KEY_MARKER}#{pointer.key}#{KEY_MARKER}#{receipt_handle}"
+    end
+
+    def embedded?(receipt_handle)
+      receipt_handle.include?(BUCKET_MARKER) && receipt_handle.include?(KEY_MARKER)
+    end
+
+    # The original SQS receipt handle, i.e. everything after the closing key marker.
+    def strip(receipt_handle)
+      return receipt_handle unless embedded?(receipt_handle)
+
+      closing = receipt_handle.index(KEY_MARKER, receipt_handle.index(KEY_MARKER) + KEY_MARKER.length)
+      receipt_handle[(closing + KEY_MARKER.length)..]
+    end
+
+    def pointer(receipt_handle)
+      PayloadPointer.new(between_markers(receipt_handle, BUCKET_MARKER), between_markers(receipt_handle, KEY_MARKER))
+    end
+
+    def between_markers(receipt_handle, marker)
+      opening = receipt_handle.index(marker)
+      closing = receipt_handle.index(marker, opening + marker.length)
+      receipt_handle[(opening + marker.length)...closing]
+    end
+    private_class_method :between_markers
+  end
+end

data/lib/sqs_extended_client/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SqsExtendedClient
+  VERSION = "0.1.0"
+end

data/lib/sqs_extended_client.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "aws-sdk-s3"
+require "aws-sdk-sqs"
+
+require_relative "sqs_extended_client/version"
+require_relative "sqs_extended_client/errors"
+require_relative "sqs_extended_client/configuration"
+require_relative "sqs_extended_client/payload_pointer"
+require_relative "sqs_extended_client/receipt_handle"
+require_relative "sqs_extended_client/payload_store"
+require_relative "sqs_extended_client/client"
+
+module SqsExtendedClient
+  def self.new(...)
+    Client.new(...)
+  end
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: sqs_extended_client
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ngoc Bui
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-09-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: A wrapper around Aws::SQS::Client that stores large message bodies in
+  S3 and sends a pointer through the queue instead, wire compatible with the AWS SQS
+  extended client libraries for Java and Python.
+email:
+- ngocbui.se@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- NOTICE
+- README.md
+- lib/sqs_extended_client.rb
+- lib/sqs_extended_client/client.rb
+- lib/sqs_extended_client/configuration.rb
+- lib/sqs_extended_client/errors.rb
+- lib/sqs_extended_client/payload_pointer.rb
+- lib/sqs_extended_client/payload_store.rb
+- lib/sqs_extended_client/receipt_handle.rb
+- lib/sqs_extended_client/version.rb
+homepage: https://github.com/xuanngoc/aws-sqs-ruby-extended-client
+licenses:
+- Apache-2.0
+metadata:
+  homepage_uri: https://github.com/xuanngoc/aws-sqs-ruby-extended-client
+  source_code_uri: https://github.com/xuanngoc/aws-sqs-ruby-extended-client
+  changelog_uri: https://github.com/xuanngoc/aws-sqs-ruby-extended-client/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Send and receive SQS messages larger than 256 KB by offloading the payload
+  to S3.
+test_files: []