sqs_simplify 0.2.0

data/lib/sqs_simplify/command.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module SqsSimplify
+  class Command
+    attr_reader :env, :queues, :with_priority, :parallel_type, :worker_size
+
+    def initialize(args = [])
+      args = [] unless args.is_a? Array
+
+      OptionParser.new do |opt|
+        opt.on('-h', '--help', 'Show help') do
+          puts opt
+          exit 1
+        end
+
+        opt.on('-n', '--number_of_workers=workers', 'Number of unique workers to spawn') do |worker_count|
+          @worker_size = build_work_count(worker_count)
+        end
+
+        opt.on('-e', '--environment=environment', 'Environment') { |env| @env = env }
+        opt.on('--queues=queues', 'queues that will be consumed') { |queues| @queues = queues.split(',') }
+        opt.on('--priority', 'with priority in the queues') { |priority| @with_priority = priority }
+        opt.on('-f', '--fork', 'parallel in processes') { @parallel_type = :in_processes }
+        opt.on('-t', '--thread', 'parallel in threads') { @parallel_type = :in_threads }
+      end.parse! args
+    end
+
+    def run(&block)
+      @running_process = true
+      block&.call(self)
+      run!
+    end
+
+    private
+
+    def build_work_count(number)
+      worker_size = number.to_i
+      worker_size.positive? && worker_size <= 11 ? worker_size : 1
+    end
+
+    def running_process?
+      @running_process
+    end
+
+    def run!
+      time = 0
+      trap('SIGINT') { stop! }
+
+      while running_process?
+        if time.zero?
+          amount_process = execute
+          time = calculate_sleep(amount_process)
+        else
+          sleep 1
+          time -= 1
+        end
+      end
+    end
+
+    def stop!
+      puts ''
+      puts 'this process will stop after consuming all messages!'
+      @running_process = false
+    end
+
+    def execute
+      worker = SqsSimplify::Worker.new(
+        queues: queues,
+        priority: with_priority,
+        worker_size: worker_size,
+        parallel_type: parallel_type
+      ) do
+        continues_the_process?
+      end
+
+      worker.perform
+    end
+
+    def calculate_sleep(amount)
+      amount.to_i.zero? ? 10 : 0
+    end
+
+    def continues_the_process?
+      running_process?
+    end
+  end
+end

data/lib/sqs_simplify/consumer.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module SqsSimplify
+  class Consumer < SqsSimplify::Base
+    include SqsSimplify::ExecutionHook
+
+    private_class_method :new
+    attr_accessor :delete_sqs_message
+    attr_reader :visibility_timeout, :errors
+
+    def initialize(sqs_message, visibility_timeout)
+      @sqs_message = sqs_message
+      @message = load_message(sqs_message)
+      @visibility_timeout = visibility_timeout
+      self.delete_sqs_message = visibility_timeout.positive?
+    end
+
+    def perform
+      raise 'Must implement this method'
+    end
+
+    def delete_sqs_message?
+      delete_sqs_message
+    end
+
+    protected
+
+    attr_reader :message, :sqs_message
+
+    class << self
+      def consume_messages(amount = nil, worker_size: nil, parallel_type: nil)
+        amount ||= maximum_message_quantity
+        @worker_size = worker_size.to_i
+        @parallel_type = parallel_type
+
+        visibility_timeout # load
+        sqs_messages = fetch_messages(amount)
+        consume_sqs_messages(sqs_messages, Time.now)
+      rescue Exception => e
+        call_hook :resolver_exception, e, sqs_messages
+      ensure
+        return sqs_messages&.count || 0
+      end
+
+      def message_not_deleted(&block)
+        hooks[:message_not_deleted] = block
+      end
+
+      def around(&block)
+        hooks[:around] = block
+      end
+
+      protected
+
+      def maximum_message_quantity
+        number = settings[:maximum_message_quantity].to_i
+        number.positive? && number <= 10 ? number : 10
+      rescue StandardError
+        10
+      end
+
+      private
+
+      def fetch_messages(amount = 10)
+        client.receive_message(
+          queue_url: queue_url,
+          message_attribute_names: ['All'], # Receive all custom attributes.
+          max_number_of_messages: amount, # Receive at most one message.
+          wait_time_seconds: 0 # Do not wait to check for the message.
+        ).messages
+      end
+
+      def consume_sqs_messages(sqs_messages, start_time)
+        call_hook(:before_all, sqs_messages)
+        benchmark { choose_process(sqs_messages, start_time) }
+        call_hook(:after_all, sqs_messages)
+      end
+
+      def choose_process(messages, start_time)
+        parallel_type, amount_processes = build_parallel_parameter(messages.size)
+
+        logger.info "Started loop with: { queue_name: #{queue_name}, #{parallel_type}: #{amount_processes}, messages: #{messages.count} }"
+        if amount_processes == 1
+          messages.each { |sqs_message| consume_sqs_message(sqs_message, start_time) }
+        else
+          Parallel.each(messages, parallel_type => amount_processes) do |sqs_message|
+            consume_sqs_message(sqs_message, start_time)
+          end
+        end
+      end
+
+      def build_parallel_parameter(messages_size)
+        process_in_parallel = messages_size > 1 &&
+                              @worker_size > 1 &&
+                              %w[in_threads in_processes].include?(@parallel_type.to_s)
+
+        process_in_parallel ? [@parallel_type, @worker_size] : [:without_parallel, 1]
+      end
+
+      def consume_sqs_message(sqs_message, start_time)
+        time_left = build_time_left(start_time)
+        consumer = new(sqs_message, time_left)
+        call_hook(:before_each, consumer)
+
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        Timeout.timeout(time_left, Errors::ExecutionExpired) do
+          around.is_a?(Proc) ? around.call(consumer) : consumer.perform
+        end
+        raise Errors::ExecutionExpired if Process.clock_gettime(Process::CLOCK_MONOTONIC) - started >= time_left
+      rescue Exception => e
+        if consumer
+          consumer.delete_sqs_message = false
+          consumer.instance_variable_set :@errors, ["Exception: #{e.message}"] + e.backtrace
+        end
+        call_hook :resolver_exception, e, consumer
+      ensure
+        call_hook(:after_each, consumer)
+        delete_message(consumer)
+      end
+
+      def build_time_left(start_time)
+        time_left = visibility_timeout - (Time.now - start_time).to_f
+        time_left = -0.1 if time_left.zero?
+        [time_left, 0.1].max
+      end
+
+      def delete_message(consumer)
+        if consumer&.delete_sqs_message
+          delete_sqs_message(consumer.send(:sqs_message))
+        else
+          call_hook :message_not_deleted, consumer
+        end
+      end
+
+      def delete_sqs_message(message)
+        client.delete_message(
+          queue_url: queue_url,
+          receipt_handle: message.receipt_handle
+        )
+        true
+      end
+
+      def benchmark(&block)
+        start_time = Time.now
+        block.call
+        logger.info "Finished in #{(Time.now - start_time).to_f} seconds"
+      end
+
+      def logger
+        SqsSimplify.logger
+      end
+    end
+  end
+end

data/lib/sqs_simplify/dead_queue.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module SqsSimplify
+  class DeadQueue < SqsSimplify::Base
+    VISIBILITY_TIMEOUT = 60 * 60
+    MESSAGE_RETENTION_PERIOD = 14 * 24 * 60 * 60
+    DELAY_SECONDS = 0
+
+    class << self
+      undef_method :max_receive_count
+
+      def main_queue
+        @main_queue ||= const_get(name.gsub('::DeadQueue', ''))
+      end
+
+      def create_queue
+        dead_queue = find_queue_by_name(queue_full_name)
+        dead_queue ||= super
+
+        # Use a redrive policy to specify the dead letter queue and its behavior.
+        redrive_policy = {
+          'maxReceiveCount' => main_queue.max_receive_count.to_s, # After the queue receives the same message 1 times, send that message to the dead letter queue.
+          'deadLetterTargetArn' => queue_arn
+        }.to_json
+
+        main_queue.update_attributes(RedrivePolicy: redrive_policy)
+        dead_queue
+      end
+
+      def send_message_to_main_queue(amount)
+        raise "Invalid argument #{amount}" unless amount.is_a?(Numeric) ||
+                                                  amount.positive?
+
+        dead_queue_to_queue_loop(amount)
+      end
+
+      private
+
+      def dead_queue_to_queue_loop(amount)
+        count = 0
+
+        loop do
+          messages = dead_queue_messages([amount, 10].min)
+          count += messages.count
+
+          messages.each do |message|
+            client.send_message(queue_url: main_queue.queue_url, message_body: message[:body])
+            client.delete_message(queue_url: queue_url, receipt_handle: message.receipt_handle)
+          end
+
+          return count if messages.count.zero? || count >= amount
+        end
+      end
+
+      def dead_queue_messages(amount)
+        client.receive_message(
+          queue_url: queue_url,
+          message_attribute_names: ['All'],
+          max_number_of_messages: amount,
+          wait_time_seconds: 0
+        ).messages
+      end
+    end
+  end
+end

data/lib/sqs_simplify/errors/execution_expired.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SqsSimplify
+  module Errors
+    class ExecutionExpired < ::Timeout::Error
+      def initialize(message = 'execution expired')
+        super
+      end
+    end
+  end
+end

data/lib/sqs_simplify/errors/non_existent_queue.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SqsSimplify
+  module Errors
+    class NonExistentQueue < StandardError
+      def initialize(queue_full_name)
+        super "The queue `#{queue_full_name}' does not exist."
+      end
+    end
+  end
+end

data/lib/sqs_simplify/errors/reserved_method_name.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SqsSimplify
+  module Errors
+    class ReservedMethodName < StandardError
+      def initialize(method)
+        super "reserved method name `#{method}'."
+      end
+    end
+  end
+end

data/lib/sqs_simplify/execution_hook.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SqsSimplify
+  module ExecutionHook
+    module ClassMethods
+      def after(type = :each, &block)
+        type = "after_#{type}".to_sym
+        hooks[type] = block
+      end
+
+      def before(type = :each, &block)
+        type = "before_#{type}".to_sym
+        hooks[type] = block
+      end
+
+      def call_hook(type, arg = nil, args = {})
+        block = hooks[type.to_sym]
+        SqsSimplify.call_hook(type, arg, args) unless self == SqsSimplify
+        block&.call(arg, args)
+      end
+
+      def hooks
+        @hooks ||= {}
+      end
+
+      def resolver_exception(&block)
+        hooks[:resolver_exception] = block
+      end
+
+      def add_hook(hook_name, *hook_names)
+        hook_names.unshift(hook_name)
+        hook_names.each do |name|
+          define_singleton_method(name) do |&block|
+            hooks[name.to_sym] = block
+          end
+        end
+      end
+    end
+
+    def self.included(base)
+      base.extend ClassMethods
+    end
+  end
+end

data/lib/sqs_simplify/faker_client.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module SqsSimplify
+  class FakerClient
+    def send_message(sqs_message)
+      id = next_id
+      url = sqs_message[:queue_url]
+      body = sqs_message[:message_body]
+      message = OpenStruct.new(receipt_handle: id, body: body, queue_url: url)
+
+      if pool[url]
+        pool[url] << message
+      else
+        pool[url] = [message]
+      end
+      OpenStruct.new(message_id: id)
+    end
+
+    def receive_message(queue_url:, max_number_of_messages:, **_args)
+      amount =
+        if max_number_of_messages.nil? || !max_number_of_messages.is_a?(Integer)
+          10
+        elsif max_number_of_messages.positive? && max_number_of_messages < 11
+          max_number_of_messages
+        end
+
+      messages = (pool[queue_url] || [])[0, amount]
+      OpenStruct.new(messages: messages)
+    end
+
+    def delete_message(queue_url:, receipt_handle:)
+      messages = pool[queue_url] || []
+      messages.delete_if do |message|
+        message.receipt_handle == receipt_handle && message.queue_url == queue_url
+      end
+
+      true
+    end
+
+    def get_queue_attributes(queue_url:, **_args)
+      messages = pool[queue_url] || []
+      OpenStruct.new(attributes: {
+                       'ApproximateNumberOfMessages' => messages.count,
+                       'VisibilityTimeout' => 60
+                     })
+    end
+
+    def get_queue_url(queue_name:)
+      OpenStruct.new queue_url: "http//sqs_faker/#{queue_name}"
+    end
+
+    private
+
+    def pool
+      self.class.pool
+    end
+
+    def next_id
+      self.class.next_id
+    end
+
+    class << self
+      def next_id
+        @next_id ||= 0
+        @next_id += 1
+        @next_id.to_s(16)
+      end
+
+      def pool
+        @pool ||= {}
+      end
+
+      def purger!
+        @pool = {}
+      end
+    end
+  end
+end

data/lib/sqs_simplify/job.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module SqsSimplify
+  class Job < SqsSimplify::Base
+    set :scheduler, true
+    private_class_method :new
+
+    def perform(*_rest, **_keyrest)
+      raise 'NotImplemented'
+    end
+
+    def new_job(**options)
+      self.class.new_job(**options)
+    end
+
+    def dump(parameters)
+      self.class.send :_dump, parameters
+    end
+
+    class << self
+      def new_job(**options)
+        self::ProxyJob.new(new, **options)
+      end
+
+      def perform(*rest, **keyrest)
+        new_job.perform(*rest, **keyrest)
+      end
+
+      def perform_later(*rest, **keyrest)
+        new_job.perform_later(*rest, **keyrest)
+      end
+
+      def consume_messages(amount = nil, **options)
+        consumer.consume_messages amount, **options
+      end
+
+      def scheduler
+        @scheduler ||= _build_scheduler
+      end
+
+      def consumer
+        @consumer ||= _build_consumer
+      end
+
+      def call_validator_parameters!(array, hash)
+        _validator_parameters.call(*array, **hash)
+      end
+
+      private
+
+      def _validator_parameters
+        @_validator_parameters ||= _build_validator_parameters
+      end
+
+      def _build_validator_parameters
+        parameters = new.method(:perform).parameters
+        params = parameters.map do |opt, name|
+          _parameters_to_args opt, name
+        end
+
+        class_eval "->(#{params.join(', ')}) {}", __FILE__, __LINE__
+      end
+
+      def _parameters_to_args(opt, name)
+        case opt
+        when :req
+          name.to_s
+        when :opt
+          "#{name}=nil"
+        when :rest
+          "*#{name}"
+        when :keyreq
+          "#{name}:"
+        when :key
+          "#{name}:nil"
+        when :keyrest
+          "**#{name}"
+        when :block
+          "&#{name}"
+        end
+      end
+
+      def inherited(sub)
+        super
+
+        if sub.superclass == SqsSimplify::Job
+          sub.set :scheduler, true
+        else
+          sub.instance_variable_set :@settings, settings.clone
+        end
+      end
+
+      def _execute(args)
+        class_name = args['class']
+        array_params, hash_params = _read_parameters args['parameters']
+
+        source = const_get(class_name)
+        instance = source.send :new
+        instance.send(:perform, *array_params, **hash_params)
+      end
+
+      def _read_parameters(parameters)
+        args = _load(parameters)
+        [args[:array], args[:hash]]
+      end
+
+      def _build_scheduler
+        klass = _build_sub_class 'Scheduler'
+        klass.private_class_method :send_message
+        klass
+      end
+
+      def _build_consumer
+        klass = _build_sub_class 'Consumer'
+        klass.define_method(:perform) { SqsSimplify::Job.send :_execute, message }
+        klass
+      end
+
+      def _build_sub_class(module_name)
+        class_eval <<~M, __FILE__, __LINE__ + 1
+          class #{self}::#{module_name}Job < SqsSimplify::#{module_name}
+            def self.settings
+              #{self}.settings
+            end
+
+            def self.client
+              #{self}.client
+            end
+
+            def self.queue_name
+              #{self}.queue_name
+            end
+          end
+        M
+
+        const_get "#{self}::#{module_name}Job"
+      end
+
+      def _dump(value)
+        Base64.encode64(Zlib::Deflate.deflate(Marshal.dump(value)))
+      end
+
+      def _load(value)
+        Marshal.load(Zlib::Inflate.inflate(Base64.decode64(value)))
+      end
+    end
+
+    class ProxyJob
+      def initialize(job, **options)
+        @job = job
+        @after = options[:after]
+        @queue_url = options[:queue_url]
+      end
+
+      def perform(*array, **hash)
+        @job.perform(*array, **hash)
+        :executed
+      end
+
+      def perform_later(*array, **hash)
+        return perform(*array, **hash) unless scheduler?
+
+        @job.class.call_validator_parameters! array, hash
+        message = {
+          'class' => @job.class.name,
+          'parameters' => @job.dump(array: array, hash: hash)
+        }
+
+        send_message message
+      end
+
+      private
+
+      def send_message(message)
+        @job.class.scheduler.send(:send_message, message: message, after: @after, queue_url: @queue_url)
+      end
+
+      def respond_to_missing?(symbol, value = nil)
+        @job.respond_to? symbol, value
+      end
+
+      def method_missing(symbol, *rest, **keyrest)
+        @job.send symbol, *rest, **keyrest
+      end
+
+      def scheduler?
+        SqsSimplify::Job.settings[:scheduler] && @job.class.settings[:scheduler]
+      end
+    end
+  end
+end

data/lib/sqs_simplify/message.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module SqsSimplify
+  class Message
+    attr_accessor :body, :delay_seconds, :queue_url, :group_id
+
+    def initialize(body:, queue_url:, delay_seconds: 0, group_id: nil)
+      self.body = body
+      self.delay_seconds = delay_seconds
+      self.queue_url = queue_url
+      self.group_id = group_id
+    end
+
+    def to_send
+      hash = {
+        queue_url: queue_url,
+        message_body: body,
+        delay_seconds: delay_seconds
+      }
+      hash[:message_group_id] = group_id.to_s if group_id
+      hash
+    end
+  end
+end