aws_sqs_extended_client 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8b2bbebadb3a245517db4d6f6e898ac88e59fb3fab3ae565352a966dce65394f
+  data.tar.gz: fc70a193700aff4e6749aeedc04cc5941d5d8f9c27824a34512321edc5f9813d
+SHA512:
+  metadata.gz: 55544c1f6dd1469429c922e5a5659621fea80eb5340582014bac635d6efb930f35a51f004f7701973c10a7d0083513d896190fb304d53caff758f7a0dc6ad6a0
+  data.tar.gz: 8470b1aa27602466c76fedbd184a5461c1dbc662a1c744e1d488840082bfe3143e66ff46395adf339832e0b31a80008712b07cc19e819148f47e17e07e249645

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Hemaraj G
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# sqs_extended_client
+
+A Ruby gem that provides support for sending and receiving large messages using Amazon SQS by offloading payloads to Amazon S3, similar to the AWS Java SDK's SQS Extended Client Library.
+
+---
+
+## ✨ Features
+
+- ✅ Automatically offloads large messages to S3
+- ✅ Configurable max message size threshold
+- ✅ Optional compression support
+- ✅ Automatic S3 payload cleanup after reading (configurable)
+- ✅ JSON schema validation (planned)
+- ✅ Error handling for missing or inaccessible buckets
+- ✅ Ready for use as a drop-in extension of `Aws::SQS::Client`
+
+---
+
+## 🔧 Installation
+
+```bash
+gem install sqs_extended_client
+
+gem 'sqs_extended_client', path: 'path/to/your/local/gem'
+
+```
+
+Example:
+
+
+```bash
+require 'aws-sdk-sqs'
+require 'aws-sdk-s3'
+require 'sqs_extended_client'
+
+sqs = Aws::SQS::Client.new(region: 'us-east-1')
+s3  = Aws::S3::Client.new(region: 'us-east-1')
+
+client = SqsExtendedClient::Client.new(
+  sqs: sqs,
+  s3: s3,
+  queue_url: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue',
+  bucket_name: 'my-sqs-large-message-bucket',
+  always_use_s3: false,
+  delete_s3_payload_after_read: true,
+  compress: true
+)
+
+client.send_message("Hello world!")
+messages = client.receive_messages
+```

data/lib/aws_sqs_extended_client/client.rb

@@ -0,0 +1,157 @@
+
+require 'aws-sdk-s3'
+require 'aws-sdk-sqs'
+require 'securerandom'
+require 'json'
+require 'zlib'
+require 'logger'
+require_relative 'errors'
+require_relative 'validator'
+require_relative 'constants'
+
+module SqsExtendedClient
+  include Constant
+  class Client
+
+    def initialize(args = {})
+
+      Validator.validate_client_options!({
+        sqs: args[:sqs], queue_url: args[:queue_url]
+      })
+
+      @sqs = args[:sqs]
+      @s3 = args[:s3]
+      @queue_url = args[:queue_url]
+      @bucket_name = args[:bucket_name]
+      @always_use_s3 = args.key?(:always_use_s3) ? args[:always_use_s3] : false
+      @max_message_size = args[:max_message_size] || Constant::DEFAULT_MAX_SQS_MESSAGE_SIZE
+      @delete_s3_payload_after_read = args.key?(:delete_s3_payload_after_read) ? args[:delete_s3_payload_after_read] : false
+      @compress = args.key?(:compress) ? args[:compress] : false
+      @logger = args[:logger] || Logger.new($stdout)
+    end
+
+    def send_message(message_body = nil)
+      store_method = Validator.validate_send_message!({
+        message_body: message_body,
+        always_use_s3: @always_use_s3,
+        max_message_size: @max_message_size,
+        bucket_name: @bucket_name,
+        s3: @s3
+      })
+
+      if store_method == :use_s3
+        key = "sqs-large/#{SecureRandom.uuid}.json"
+        data = @compress ? Zlib::Deflate.deflate(message_body) : message_body
+
+        begin
+          @s3.put_object(bucket: @bucket_name, key: key, body: data)
+          @logger&.info("Uploaded large message to S3: #{key}")
+        rescue Aws::S3::Errors::NoSuchBucket => e
+          raise BucketNotFoundError.new(@bucket_name, cause: e)
+        rescue Aws::S3::Errors::AccessDenied => e
+          raise AccessDeniedError.new("Access denied to bucket: #{@bucket_name}", cause: e)
+        rescue Aws::S3::Errors::ServiceError => e
+          raise S3UploadError.new("Failed to upload to S3: #{e.message}", cause: e)
+        rescue StandardError => e
+          @logger&.error("Unexpected S3 error while uploading: #{e.class} - #{e.message}")
+          raise S3UploadError.new("Unexpected error during S3 upload", cause: e)
+        end
+
+        body = {
+          s3_pointer: { bucket: @bucket_name, key: key },
+          compressed: @compress
+        }.to_json
+      else
+        body = message_body
+      end
+
+      begin
+        @sqs.send_message(queue_url: @queue_url, message_body: body)
+        @logger&.info("Sent message to SQS#{' (S3 pointer)' if store_method == :use_s3}")
+      rescue Aws::SQS::Errors::ServiceError => e
+        @logger&.error("Failed to send message to SQS: #{e.class} - #{e.message}")
+        raise SqsSendError.new("SQS send failed: #{e.message}", cause: e)
+      rescue StandardError => e
+        @logger&.error("Unexpected error during SQS send: #{e.class} - #{e.message}")
+        raise SqsSendError.new("Unexpected error during SQS send", cause: e)
+      end
+    end
+
+
+    def receive_messages(args = {})
+      max_messages = args[:max_number] || Constant::MAX_MESSAGES
+      wait_time = args[:wait_time_seconds] || Constant::LONG_POLL_WAIT_TIME
+      auto_delete = args.key?(:auto_delete) ? args[:auto_delete] : true
+
+      resp = @sqs.receive_message(
+        queue_url: @queue_url,
+        max_number_of_messages: max_messages,
+        wait_time_seconds: wait_time,
+        message_attribute_names: ["All"]
+      )
+
+      return [] if resp.messages.empty?
+
+      resp.messages.map do |msg|
+        begin
+          decoded_body = parse_message_body(msg)
+          msg.define_singleton_method(:decoded_body) { decoded_body }
+
+          msg
+        rescue JSON::ParserError => e
+          @logger&.error("Failed to parse JSON from message #{msg.message_id}: #{e.message}")
+          nil
+        rescue Aws::S3::Errors::NoSuchKey => e
+          @logger&.error("S3 object not found for message #{msg.message_id}: #{e.message}")
+          nil
+        rescue Aws::S3::Errors::NoSuchBucket => e
+          @logger&.error("S3 bucket not found for message #{msg.message_id}: #{e.message}")
+          nil
+        rescue Aws::S3::Errors::AccessDenied => e
+          @logger&.error("Access denied for S3 object in message #{msg.message_id}: #{e.message}")
+          nil
+        rescue Aws::S3::Errors::ServiceError => e
+          @logger&.error("S3 service error while reading message #{msg.message_id}: #{e.message}")
+          nil
+        rescue Zlib::DataError => e
+          @logger&.error("Decompression failed for message #{msg.message_id}: #{e.message}")
+          nil
+        rescue StandardError => e
+          @logger&.error("Unexpected error in message #{msg.message_id}: #{e.class} - #{e.message}")
+          nil
+        ensure
+          if auto_delete
+            @sqs.delete_message(queue_url: @queue_url, receipt_handle: msg.receipt_handle)
+            @logger&.info("Deleted SQS message: #{msg.message_id}")
+          end  
+        end
+      end.compact
+    end
+
+    private
+
+    def parse_message_body(msg)
+      parsed = JSON.parse(msg.body)
+
+      if parsed && parsed["s3_pointer"]
+        bucket = parsed["s3_pointer"]["bucket"]
+        key = parsed["s3_pointer"]["key"]
+        compressed = parsed["compressed"]
+
+        s3_data = @s3.get_object(bucket: bucket, key: key).body.read
+        result = compressed ? Zlib::Inflate.inflate(s3_data) : s3_data
+
+        p "result #{result}"
+
+        if @delete_s3_payload_after_read
+          @s3.delete_object(bucket: bucket, key: key)
+          @logger&.info("Deleted S3 object after read: #{key}")
+        end
+
+        result
+      else
+        msg.body
+      end
+    end
+  end
+end

data/lib/aws_sqs_extended_client/constants.rb

@@ -0,0 +1,7 @@
+module SqsExtendedClient
+	module Constant
+	  DEFAULT_MAX_SQS_MESSAGE_SIZE = 256 * 1024.freeze
+	  MAX_MESSAGES = 10.freeze
+      LONG_POLL_WAIT_TIME = 20.freeze
+    end
+end

data/lib/aws_sqs_extended_client/errors.rb

@@ -0,0 +1,44 @@
+module SqsExtendedClient
+  class Error < StandardError
+    attr_reader :cause
+
+    def initialize(msg = nil, cause: nil)
+      @cause = cause
+      super(msg)
+    end
+  end
+
+  class BucketNameMissingError < Error
+    def initialize
+      super("S3 bucket name must be provided but was nil or empty.")
+    end
+  end
+
+  class BucketNotFoundError < Error
+    attr_reader :bucket_name
+
+    def initialize(bucket_name, cause: nil)
+      @bucket_name = bucket_name
+      msg = "S3 bucket not found or not accessible: #{bucket_name}"
+      super(msg, cause: cause)
+    end
+  end
+
+  class AccessDeniedError < Error
+    def initialize(msg = "Access denied to the S3 resource", cause: nil)
+      super(msg, cause: cause)
+    end
+  end
+
+  class S3UploadError < Error
+    def initialize(msg = "Unexpected error while uploading to S3", cause: nil)
+      super(msg, cause: cause)
+    end
+  end
+
+  class SqsSendError < Error
+    def initialize(msg = "SQS send_message failed", cause: nil)
+      super(msg, cause: cause)
+    end
+  end
+end

data/lib/aws_sqs_extended_client/validator.rb

@@ -0,0 +1,39 @@
+
+require_relative 'errors'
+
+module SqsExtendedClient
+  class Validator
+    REQUIRED_ATTRIBUTES = %i[sqs queue_url].freeze
+    S3_MESSASGE_ATTRIBUTES = %i[s3 bucket_name].freeze
+    # RECEIVE_MESSAGE_ATTRIBUTES = 
+
+    def self.validate_client_options!(options)
+      missing = REQUIRED_ATTRIBUTES.select do |key|
+        options[key].nil? || options[key].to_s.strip.empty?
+      end
+
+      unless missing.empty?
+        raise ArgumentError, "Missing required attribute(s): #{missing.join(', ')}"
+      end
+    end
+
+    def self.validate_send_message!(options)
+      raise ArgumentError, "Missing required attribute: message_body" if options[:message_body].nil?
+
+      raise ArgumentError, "max_message_size must be a positive integer" if options[:max_message_size] && options[:max_message_size].to_i <= 0
+
+      if options[:always_use_s3] || options[:message_body].bytesize > options[:max_message_size]
+        missing = S3_MESSASGE_ATTRIBUTES.select do |key|
+          options[key].nil? || options[key].to_s.strip.empty?
+        end
+
+        unless missing.empty?
+          raise ArgumentError, "Missing required attribute(s): #{missing.join(', ')}"
+        end
+        return :use_s3
+      else
+        return :use_sqs
+      end
+    end
+  end
+end

data/lib/aws_sqs_extended_client.rb

@@ -0,0 +1,4 @@
+
+require_relative "aws_sqs_extended_client/client"
+require_relative "aws_sqs_extended_client/errors"
+require_relative "aws_sqs_extended_client/validator"

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: aws_sqs_extended_client
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Hemaraj G
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-08-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby gem that mimics AWS SQS Extended Client with features like S3 offloading,
+  compression, cleanup, validation, and logging.
+email:
+- hemarajg7@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- lib/aws_sqs_extended_client.rb
+- lib/aws_sqs_extended_client/client.rb
+- lib/aws_sqs_extended_client/constants.rb
+- lib/aws_sqs_extended_client/errors.rb
+- lib/aws_sqs_extended_client/validator.rb
+homepage: https://github.com/hemarajg/ruby_aws_extended_client
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Extended SQS client with S3 offloading and validation.
+test_files: []