sqrewdriver 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 960177a8b086b8f3a761255650cd099772374d5a775fa0eaddadbc7c11d0a4e6
+  data.tar.gz: 0a187eac2691915e97ff7fa321aeef27bef3d4b3aeeee11826e310c34108ea71
+SHA512:
+  metadata.gz: f5ffdc6396c6fca90c6c33cb385da8a7df5dff42f536dcf3716bca5733f7b7e1c1ddccbcc4957211666760c837f2550871daec05650558a9cbc917fc036d4fc1
+  data.tar.gz: 7ffd2951ebf5171c0821dcf1ec71a34a96953d8f3e61f17c92291104aeee33372b9d13c697be7411e30ef22913e99832b563dd2352338a9c3d2164eb44a4e0c1

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.1
+before_install: gem install bundler -v 2.0.1

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in sqrewdriver.gemspec
+gemspec
+
+gem "oj"

data/Gemfile.lock

@@ -0,0 +1,52 @@
+PATH
+  remote: .
+  specs:
+    sqrewdriver (0.1.0)
+      aws-sdk-sqs
+      concurrent-ruby
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.0.1)
+    aws-partitions (1.141.0)
+    aws-sdk-core (3.46.2)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-sqs (1.10.0)
+      aws-sdk-core (~> 3, >= 3.39.0)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.3)
+    concurrent-ruby (1.1.4)
+    diff-lcs (1.3)
+    jmespath (1.4.0)
+    oj (3.7.9)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  oj
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  sqrewdriver!
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 joker1007
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,84 @@
+# sqrewdriver
+
+This gem extend Amazon SQS client. (parallelize and aggregation).
+
+## Features
+
+- Message buffering and Use SendMessageBatch implicitly
+- Sending message on thread pool by [concurrent-ruby](https://github.com/ruby-concurrency/concurrent-ruby)
+- Message Aggregation into serialized array
+- Serialize message implicitly (default: JSON format)
+- Currently, Support only `send_message_buffered`
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sqrewdriver'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sqrewdriver
+
+## Usage
+
+```ruby
+client = Sqrewdriver::Client.new(queue_url: "https://sqs.ap-northeast-1.amazonaws.com/dummy/dummy")
+
+# or
+
+sqs = Aws::SQS::Client.new(log_level: :debug)
+client = Sqrewdriver::Client.new(
+  queue_url: "https://sqs.ap-northeast-1.amazonaws.com/dummy/dummy",
+  client: sqs
+)
+
+# If use message aggregation
+
+client = Sqrewdriver::Client.new(
+  queue_url: "https://sqs.ap-northeast-1.amazonaws.com/dummy/dummy",
+  aggregate_messages_per: 100
+) # one SQS message is a serialized array having 100 items.
+
+payload_data.each do |d|
+  client.send_message_buffered(message_body: d)
+end
+
+client.flush # Don't forget to call flush finally
+
+# If any exception occurs when client is flushing messages,
+# client raises `Sqrewdriver::SendMessageErrors`
+```
+
+### Change serializer
+
+Serializer needs `dump(value)` method.
+
+```ruby
+client = Sqrewdriver::Client.new(
+  queue_url: "https://sqs.ap-northeast-1.amazonaws.com/dummy/dummy",
+  serializer: YAML
+)
+```
+
+If you want to pass options to serializer, Please implement wrapping class.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/joker1007/sqrewdriver.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "sqrewdriver"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/sqrewdriver.rb

@@ -0,0 +1,20 @@
+require "sqrewdriver/version"
+require "sqrewdriver/serdes/json_serde"
+
+module Sqrewdriver
+  class << self
+    # @return [#dump]
+    def default_serializer
+      @default_serializer
+    end
+
+    # @param serializer [#dump]
+    def default_serializer=(serializer)
+      @default_serializer = serializer
+    end
+  end
+
+  self.default_serializer = Serdes::JSONSerde.new
+end
+
+require "sqrewdriver/client"

data/lib/sqrewdriver/client.rb

@@ -0,0 +1,208 @@
+require "set"
+require "monitor"
+require "concurrent"
+require "sqrewdriver/errors"
+require "aws-sdk-sqs"
+
+module Sqrewdriver
+  class Client
+    MAX_BATCH_SIZE = 10
+    MAX_PAYLOAD_SIZE = 256 * 1024
+
+    def initialize(queue_url:, client: nil, threads: 32, serializer: Sqrewdriver.default_serializer, aggregate_messages_per: nil, flush_retry_count: 5, **options)
+      if client
+        @client = client
+      else
+        @client = Aws::SQS::Client.new(options)
+      end
+
+      @queue_url = queue_url
+      @message_buffer = Concurrent::Array.new
+      @thread_pool = Concurrent::FixedThreadPool.new(threads)
+      @flush_retry_count = flush_retry_count
+      @waiting_futures = Concurrent::Set.new
+      @flush_mutex = Mutex.new
+      @aggregate_messages_per = aggregate_messages_per
+
+      ensure_serializer_for_aggregation!(serializer)
+
+      @sending_buffer = SendingBuffer.new(client: @client, queue_url: queue_url, serializer: serializer, thread_pool: @thread_pool)
+    end
+
+    # Add a message to buffer.
+    #
+    # If count of buffered messages exceed 10 or aggregate_messages_per
+    # else if sum of message size exceeds 256KB,
+    # send payload to SQS asynchronously.
+    def send_message_buffered(queue_url: nil, **params)
+      add_message_to_buffer(params)
+
+      if need_flush?
+        flush_async
+      end
+    end
+
+    class SendingBuffer
+      class Chunk
+        attr_reader :data, :bytesize
+
+        def initialize
+          @data = []
+          @bytesize = 0
+        end
+
+        def add(message, size)
+          @data << message
+          @bytesize += size
+        end
+      end
+
+      include MonitorMixin
+
+      attr_reader :chunks
+
+      def initialize(client:, queue_url:, serializer:, thread_pool:)
+        super()
+        @client = client
+        @queue_url = queue_url
+        @chunks = Concurrent::Array.new
+        @serializer = serializer
+        @thread_pool = thread_pool
+      end
+
+      def add_message(message)
+        serialized = @serializer.dump(message[:message_body])
+        message[:message_body] = serialized
+        add_size = calculate_message_size(serialized, message[:attributes])
+
+        synchronize do
+          @chunks << Chunk.new if @chunks.empty?
+          if @chunks.last.bytesize + add_size > MAX_PAYLOAD_SIZE
+            new_chunk = Chunk.new
+            new_chunk.add(message, add_size)
+            @chunks << new_chunk
+          else
+            @chunks.last.add(message, add_size)
+          end
+        end
+      end
+
+      def add_aggregated_messages(messages)
+        base_message = messages[0]
+
+        message_bodies = messages.map { |m| m[:message_body] }
+        serialized = @serializer.dump(message_bodies)
+        base_message[:message_body] = serialized
+        add_size = calculate_message_size(serialized, base_message[:message_attributes])
+
+        synchronize do
+          @chunks << Chunk.new if @chunks.empty?
+          if @chunks.last.bytesize + add_size > MAX_PAYLOAD_SIZE
+            new_chunk = Chunk.new
+            new_chunk.add(base_message, add_size)
+            @chunks << new_chunk
+          else
+            @chunks.last.add(base_message, add_size)
+          end
+        end
+      end
+
+      def has_full_chunk?
+        @chunks.size > 1
+      end
+
+      def send_first_chunk_async
+        Concurrent::Promises.future_on(@thread_pool, @chunks) do |chunks|
+          sending = chunks.shift
+          if sending
+            sending.data.each_with_index do |params, idx|
+              params[:id] = idx.to_s
+              params
+            end
+            send_message_batch_with_retry(entries: sending.data)
+          end
+        end
+      end
+
+      private
+
+      def calculate_message_size(body, attributes)
+        sum = body.bytesize
+        attributes&.each do |n, a|
+          sum += n.bytesize
+          sum += a[:data_type].bytesize
+          sum += a[:string_value].bytesyze
+          sum += a[:binary_value].bytesyze
+          sum += a[:string_list_values].sum(&:bytesize)
+          sum += a[:binaly_list_values].sum(&:bytesize)
+        end
+        sum
+      end
+
+      def send_message_batch_with_retry(entries:, retry_count: 0)
+        resp = @client.send_message_batch(queue_url: @queue_url, entries: entries)
+
+        unless resp.failed.empty?
+          raise SendMessageBatchFailure.new(resp.failed) if retry_count >= @flush_retry_count
+
+          failed_entries = resp.failed.map { |f| entries[f.id.to_i] }
+          send_message_batch_with_retry(entries: failed_entries, retry_count: retry_count + 1)
+        end
+      end
+    end
+
+    def flush_async
+      until @message_buffer.empty? do
+        if @aggregate_messages_per
+          messages = @message_buffer.shift(@aggregate_messages_per)
+          @sending_buffer.add_aggregated_messages(messages)
+        else
+          message = @message_buffer.shift
+          @sending_buffer.add_message(message)
+        end
+
+        if @sending_buffer.has_full_chunk?
+          send_first_chunk_async
+        end
+      end
+
+      send_first_chunk_async
+    end
+
+    def wait_flushing(timeout = nil)
+      zipped = Concurrent::Promises.zip_futures_on(@thread_pool, *@waiting_futures)
+      exceptions = zipped.reason(timeout)
+      raise Sqrewdriver::SendMessageErrors.new(exceptions) if exceptions
+    end
+
+    def flush
+      flush_async
+      wait_flushing
+    end
+
+    private
+
+    def add_message_to_buffer(params)
+      @message_buffer << params
+    end
+
+    def need_flush?
+      @message_buffer.length >= (@aggregate_messages_per&.*(10) || 10)
+    end
+
+    def send_first_chunk_async
+      future = @sending_buffer.send_first_chunk_async 
+      @waiting_futures << future
+      future.on_resolution_using(@thread_pool) do |fulfilled, value, reason|
+        @waiting_futures.delete(future)
+      end
+    end
+
+    def ensure_serializer_for_aggregation!(serializer)
+      valid_serializer = @aggregate_messages_per.nil? || serializer.is_a?(Sqrewdriver::Serdes::JSONSerde)
+      unless valid_serializer
+        raise InvalidSerializer, "If you use `aggregate_messages_per`, serializer must be `Oj` or `MultiJson`"
+      end
+    end
+  end
+end

data/lib/sqrewdriver/errors.rb

@@ -0,0 +1,21 @@
+module Sqrewdriver
+  class SendMessageErrors < StandardError
+    attr :errors
+
+    def initialize(errors)
+      @errors = errors
+    end
+
+    def messages
+      @errors.map(&:message)
+    end
+  end
+
+  class SendMessageBatchFailure < StandardError
+    attr_reader :failed
+
+    def initialize(failed)
+      @failed = failed
+    end
+  end
+end

data/lib/sqrewdriver/serdes/json_serde.rb

@@ -0,0 +1,42 @@
+module Sqrewdriver
+  module Serdes
+    class JSONSerde
+      class JSONWrapper
+        def dump(val, **options)
+          JSON.generate(val, **options)
+        end
+
+        def load(val, **options)
+          JSON.load(val, nil, **options)
+        end
+      end
+
+      def initialize(backend: :json, serialize_options: {}, deserialize_options: {})
+        @serialize_options = serialize_options
+        @deserialize_options = deserialize_options
+        @backend = lookup_json_backend(backend)
+      end
+
+      def dump(val)
+        @backend.dump(val, **@serialize_options)
+      end
+
+      def load(val)
+        @backend.load(val, **@deserialize_options)
+      end
+
+      private
+
+      def lookup_json_backend(name)
+        case name
+        when :json
+          require "json"
+          JSONWrapper.new
+        when :oj
+          require "oj"
+          Oj
+        end
+      end
+    end
+  end
+end

data/lib/sqrewdriver/serdes/raw_serde.rb

@@ -0,0 +1,16 @@
+module Sqrewdriver
+  module Serdes
+    class RawSerde
+      def initialize(serialize_options: {}, deserialize_options: {})
+      end
+
+      def dump(val)
+        val
+      end
+
+      def load(val)
+        val
+      end
+    end
+  end
+end

data/lib/sqrewdriver/version.rb

@@ -0,0 +1,3 @@
+module Sqrewdriver
+  VERSION = "0.1.0"
+end

data/sqrewdriver.gemspec

@@ -0,0 +1,31 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "sqrewdriver/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "sqrewdriver"
+  spec.version       = Sqrewdriver::VERSION
+  spec.authors       = ["joker1007"]
+  spec.email         = ["kakyoin.hierophant@gmail.com"]
+
+  spec.summary       = %q{This gem extend Amazon SQS client. (parallelize and aggregation)}
+  spec.description   = %q{This gem extend Amazon SQS client. (parallelize and aggregation)}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "aws-sdk-sqs"
+  spec.add_runtime_dependency "concurrent-ruby"
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: sqrewdriver
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- joker1007
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-03-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: This gem extend Amazon SQS client. (parallelize and aggregation)
+email:
+- kakyoin.hierophant@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/sqrewdriver.rb
+- lib/sqrewdriver/client.rb
+- lib/sqrewdriver/errors.rb
+- lib/sqrewdriver/serdes/json_serde.rb
+- lib/sqrewdriver/serdes/raw_serde.rb
+- lib/sqrewdriver/version.rb
+- sqrewdriver.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: This gem extend Amazon SQS client. (parallelize and aggregation)
+test_files: []