toiler 0.6.1 → 0.7.0

data/lib/toiler/actor/supervisor.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'toiler/actor/fetcher'
 require 'toiler/actor/processor'
 
@@ -7,10 +9,9 @@ module Toiler
     class Supervisor < Concurrent::Actor::RestartingContext
       include Utils::ActorLogging
 
-      attr_accessor :client
-
       def initialize
-        @client = ::Aws::SQS::Client.new
+        super
+
         spawn_processors
         spawn_fetchers
       end
@@ -21,11 +22,12 @@ module Toiler
 
       def spawn_fetchers
         Toiler.active_worker_class_registry.each do |queue, klass|
-          count = klass.concurrency
+          count           = klass.concurrency
+          provider        = klass.provider
           begin
             fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
                                             supervise: true,
-                                            args: [queue, client, count]
+                                            args: [queue, count, provider]
             Toiler.set_fetcher queue, fetcher
           rescue StandardError => e
             error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"

data/lib/toiler/actor/utils/actor_logging.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 module Toiler
   module Actor
     module Utils
       # Provides helper methods for logging
       module ActorLogging
         def error(msg)
-          log Logger::Severity::ERROR,  msg
+          log Logger::Severity::ERROR, msg
         end
 
         def info(msg)
@@ -12,7 +14,7 @@ module Toiler
         end
 
         def debug(msg)
-          log Logger::Severity::DEBUG,  msg
+          log Logger::Severity::DEBUG, msg
         end
 
         def warn(msg)
@@ -20,7 +22,7 @@ module Toiler
         end
 
         def fatal(msg)
-          log Logger::Severity::FATAL,  msg
+          log Logger::Severity::FATAL, msg
         end
       end
     end

data/lib/toiler/aws/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Toiler
   module Aws
     # SQS Message abstraction
@@ -24,7 +26,7 @@ module Toiler
         )
       end
 
-      def visibility_timeout=(timeout)
+      def modify_ack_deadline!(timeout)
         client.change_message_visibility(
           queue_url: queue_url,
           receipt_handle: data.receipt_handle,

data/lib/toiler/aws/queue.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'toiler/aws/message'
 
 module Toiler
@@ -7,13 +9,13 @@ module Toiler
     class Queue
       attr_accessor :name, :client, :url
 
-      def initialize(name, client = nil)
+      def initialize(name, client)
         @name   = name
-        @client = client || ::Aws::SQS::Client.new
+        @client = client
         @url    = client.get_queue_url(queue_name: name).queue_url
       end
 
-      def visibility_timeout
+      def ack_deadline
         client.get_queue_attributes(
           queue_url: url,
           attribute_names: ['VisibilityTimeout']
@@ -30,14 +32,22 @@ module Toiler
 
       def send_messages(options)
         client.send_message_batch(
-          sanitize_message_body options.merge queue_url: url
+          sanitize_message_body(options.merge(queue_url: url))
         )
       end
 
-      def receive_messages(options)
-        client.receive_message(options.merge(queue_url: url))
-          .messages
-          .map { |m| Message.new(client, url, m) }
+      def receive_messages(wait: nil, max_messages: nil)
+        client.receive_message(attribute_names: %w[All],
+                               message_attribute_names: %w[All],
+                               wait_time_seconds: wait,
+                               max_number_of_messages: max_messages,
+                               queue_url: url)
+              .messages
+              .map { |m| Message.new(client, url, m) }
+      end
+
+      def max_messages
+        10
       end
 
       private
@@ -50,7 +60,7 @@ module Toiler
           if body.is_a?(Hash)
             m[:message_body] = JSON.dump(body)
           elsif !body.is_a? String
-            fail ArgumentError, "Body must be a String, found #{body.class}"
+            raise ArgumentError, "Body must be a String, found #{body.class}"
           end
         end
 

data/lib/toiler/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'singleton'
 require 'timeout'
 require 'optparse'
@@ -7,6 +9,7 @@ module Toiler
   # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
   class Shutdown < Interrupt; end
 
+  # WaitShutdown is used to handle graceful shutdowns
   class WaitShutdown < Interrupt
     attr_accessor :wait
 
@@ -39,12 +42,12 @@ module Toiler
     private
 
     def handle_stop
-      while (readable_io = IO.select([@self_read]))
+      while (readable_io = @self_read.wait_readable)
         handle_signal(readable_io.first[0].gets.strip)
       end
-    rescue WaitShutdown => shutdown_error
-      Toiler.logger.info "Received Interrupt, Waiting up to #{shutdown_error.wait} seconds for actors to finish..."
-      success = supervisor.ask(:terminate!).wait(shutdown_error.wait)
+    rescue WaitShutdown => e
+      Toiler.logger.info "Received Interrupt, Waiting up to #{e.wait} seconds for actors to finish..."
+      success = supervisor.ask(:terminate!).wait(e.wait)
       if success
         Toiler.logger.info 'Supervisor successfully terminated'
       else
@@ -58,6 +61,7 @@ module Toiler
       Concurrent.global_fast_executor.shutdown
       Concurrent.global_io_executor.shutdown
       return if Concurrent.global_io_executor.wait_for_termination(60)
+
       Concurrent.global_io_executor.kill
     end
 
@@ -67,21 +71,20 @@ module Toiler
     end
 
     def trap_signals
-      %w(INT TERM QUIT USR1 USR2 TTIN ABRT).each do |sig|
-        begin
-          trap sig do
-            @self_write.puts(sig)
-          end
-        rescue ArgumentError
-          puts "System does not support signal #{sig}"
+      %w[INT TERM QUIT USR1 USR2 TTIN ABRT].each do |sig|
+        trap sig do
+          @self_write.puts(sig)
         end
+      rescue ArgumentError
+        puts "System does not support signal #{sig}"
       end
     end
 
     def print_stacktraces
       return unless Toiler.logger
-      Toiler.logger.info "-------------------"
-      Toiler.logger.info "Received QUIT, dumping threads:"
+
+      Toiler.logger.info '-------------------'
+      Toiler.logger.info 'Received QUIT, dumping threads:'
       Thread.list.each do |t|
         id = t.object_id
         Toiler.logger.info "[thread:#{id}] #{t.backtrace.join("\n[thread:#{id}] ")}"
@@ -91,21 +94,24 @@ module Toiler
 
     def print_status
       return unless Toiler.logger
-      Toiler.logger.info "-------------------"
-      Toiler.logger.info "Received QUIT, dumping status:"
+
+      Toiler.logger.info '-------------------'
+      Toiler.logger.info 'Received QUIT, dumping status:'
       Toiler.queues.each do |queue|
         fetcher = Toiler.fetcher(queue).send(:core).send(:context)
         processor_pool = Toiler.processor_pool(queue).send(:core).send(:context)
-        processors = processor_pool.instance_variable_get(:@workers).collect{|w| w.send(:core).send(:context)}
-        busy_processors = processors.count{|pr| pr.executing?}
+        processors = processor_pool.instance_variable_get(:@workers).collect { |w| w.send(:core).send(:context) }
+        busy_processors = processors.count(&:executing?)
         message = "Status for [queue:#{queue}]:"
-        message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [waiting_messages:#{fetcher.waiting_messages}] [free_processors:#{fetcher.free_processors}]"
+        message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] " \
+                   "[waiting_messages:#{fetcher.waiting_messages}] [free_processors:#{fetcher.free_processors}] " \
+                   "[scheduled_task:#{!fetcher.scheduled_task.nil?}]"
         message += "\n[processor_pool:#{processor_pool.name}] [workers:#{processors.count}] [busy:#{busy_processors}]"
         processors.each do |processor|
           thread = processor.thread
-          thread_id = thread.nil? ? "nil" : thread.object_id
+          thread_id = thread.nil? ? 'nil' : thread.object_id
           message += "\n[processor:#{processor.name}] [executing:#{processor.executing?}] [thread:#{thread_id}]"
-          message += " Stack:\n" + thread.backtrace.join("\n\t") unless thread.nil?
+          message += " Stack:\n#{thread.backtrace.join("\n\t")}" unless thread.nil?
         end
         Toiler.logger.info message
       end
@@ -118,22 +124,24 @@ module Toiler
         print_stacktraces
         print_status
       when 'INT', 'TERM'
-        fail WaitShutdown, 60
+        raise WaitShutdown, 60
       when 'ABRT'
-        fail WaitShutdown, Toiler.options[:shutdown_timeout] * 60
+        raise WaitShutdown, Toiler.options[:shutdown_timeout] * 60
       end
     end
 
     def load_concurrent
       require 'concurrent-edge'
-      Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
-        Toiler.logger.log(level, msg, progname, &block)
-      end if Toiler.logger
+      if Toiler.logger
+        Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
+          Toiler.logger.log(level, msg, progname, &block)
+        end
+      end
     end
 
     def daemonize
       return unless Toiler.options[:daemon]
-      fail 'Logfile required when daemonizing' unless Toiler.options[:logfile]
+      raise 'Logfile required when daemonizing' unless Toiler.options[:logfile]
 
       files_to_reopen = []
       ObjectSpace.each_object(File) do |file|
@@ -148,12 +156,10 @@ module Toiler
 
     def reopen_files(files_to_reopen)
       files_to_reopen.each do |file|
-        begin
-          file.reopen file.path, 'a+'
-          file.sync = true
-        rescue StandardError
-          puts "Failed to reopen file #{file}"
-        end
+        file.reopen file.path, 'a+'
+        file.sync = true
+      rescue StandardError
+        puts "Failed to reopen file #{file}"
       end
     end
 

data/lib/toiler/gcp/message.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Toiler
+  module Gcp
+    # PubSub Message abstraction
+    # Provides methods for querying and acting on a PubSub message
+    class Message
+      attr_accessor :message
+
+      def initialize(message)
+        @message = message
+      end
+
+      def delete
+        message.acknowledge!
+      end
+
+      def modify_ack_deadline!(timeout)
+        message.modify_ack_deadline! timeout
+      end
+
+      def message_id
+        message.message_id
+      end
+
+      def body
+        message.data
+      end
+
+      def attributes
+        message.attributes
+      end
+
+      def delivery_attempt
+        message.delivery_attempt
+      end
+
+      def ordering_key
+        message.ordering_key
+      end
+
+      def reject!
+        message.reject!
+      end
+
+      def published_at
+        message.published_at
+      end
+
+      def ack_id
+        message.ack_id
+      end
+    end
+  end
+end

data/lib/toiler/gcp/queue.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'toiler/gcp/message'
+
+module Toiler
+  module Gcp
+    # GCP PubSub Queue abstraction
+    # Provides methods for querying and acting on a PubSub queue
+    class Queue
+      attr_accessor :name, :subscription
+
+      def initialize(name, client)
+        @name = name
+        @subscription = client.subscription name, skip_lookup: true
+      end
+
+      def ack_deadline
+        subscription.deadline
+      end
+
+      def delete_messages(messages)
+        subscription.acknowledge(messages)
+      end
+
+      def max_messages
+        # pubsub limit is 1000, but it makes little sense to pull so many messages at once
+        100
+      end
+
+      def receive_messages(wait: nil, max_messages: nil)
+        immediate = wait.nil? || wait.zero?
+        subscription.pull(immediate: immediate, max: max_messages)
+                    .map { |m| Message.new(m) }
+      end
+    end
+  end
+end

data/lib/toiler/utils/argument_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Toiler
   module Utils
     # Parses command-line arguments

data/lib/toiler/utils/environment_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'erb'
 require 'yaml'
 
@@ -12,7 +14,7 @@ module Toiler
       end
 
       def self.load_for_rails_console
-        load(config_file: (Rails.root + 'config' + 'toiler.yml'))
+        load(config_file: "#{Rails.root}configtoiler.yml")
       end
 
       def initialize(options)
@@ -26,36 +28,32 @@ module Toiler
         Toiler.options.merge!(config_file_options)
         Toiler.options.merge!(options)
         initialize_aws
+        initialize_gcp
       end
 
       private
 
       def config_file_options
-        if (path = options[:config_file])
-          unless File.exist?(path)
-            Toiler.logger.warn "Config file #{path} does not exist"
-            path = nil
-          end
+        if (path = options[:config_file]) && !File.exist?(path)
+          Toiler.logger.warn "Config file #{path} does not exist"
+          path = nil
         end
 
         return {} unless path
 
-        deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
+        deep_symbolize_keys YAML.safe_load(ERB.new(File.read(path)).result)
       end
 
       def initialize_aws
         return if Toiler.options[:aws].empty?
-        ::Aws.config[:region] = Toiler.options[:aws][:region]
-        ::Aws.config[:endpoint] = Toiler.options[:aws][:endpoint] if Toiler.options[:aws][:endpoint]
-        set_aws_credentials
+
+        Toiler.aws_client = ::Aws::SQS::Client.new Toiler.options[:aws]
       end
 
-      def set_aws_credentials
-        return unless Toiler.options[:aws][:access_key_id]
-        ::Aws.config[:credentials] = ::Aws::Credentials.new(
-          Toiler.options[:aws][:access_key_id],
-          Toiler.options[:aws][:secret_access_key]
-        )
+      def initialize_gcp
+        return if Toiler.options[:gcp].empty?
+
+        Toiler.gcp_client = ::Google::Cloud::PubSub.new Toiler.options[:gcp]
       end
 
       def initialize_logger
@@ -90,8 +88,8 @@ module Toiler
         require options[:require]
       end
 
-      def deep_symbolize_keys(h)
-        h.each_with_object({}) do |(key, value), result|
+      def deep_symbolize_keys(hash)
+        hash.each_with_object({}) do |(key, value), result|
           k = key.respond_to?(:to_sym) ? key.to_sym : key
           result[k] = if value.is_a? Hash
                         deep_symbolize_keys value

data/lib/toiler/utils/logging.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'time'
 require 'logger'
 
@@ -21,8 +23,8 @@ module Toiler
 
       module_function
 
-      def initialize_logger(log_target = STDOUT)
-        log_target = STDOUT if log_target.nil?
+      def initialize_logger(log_target = $stdout)
+        log_target = $stdout if log_target.nil?
         @logger = Logger.new(log_target)
         @logger.level = Logger::INFO
         @logger.formatter = Pretty.new
@@ -34,7 +36,7 @@ module Toiler
       end
 
       def logger=(log)
-        @logger = (log ? log : Logger.new('/dev/null'))
+        @logger = (log || Logger.new('/dev/null'))
       end
     end
   end

data/lib/toiler/version.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 # Toiler Version
 module Toiler
-  VERSION = '0.6.1'.freeze
+  VERSION = '0.7.0'
 end

data/lib/toiler/worker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Toiler
   # Toiler's Worker behaviour
   module Worker
@@ -34,26 +36,32 @@ module Toiler
     module ClassMethods
       def toiler_options(options = {})
         return class_variable_get(:@@toiler_options) if options.empty?
+
         Toiler.register_worker(options[:queue], self) if options[:queue]
         class_variable_get(:@@toiler_options).merge! options
       end
 
-      def batch?
-        class_variable_get(:@@toiler_options)[:batch]
-      end
-
       def concurrency
         class_variable_get(:@@toiler_options)[:concurrency]
       end
 
+      def provider
+        class_variable_get(:@@toiler_options)[:provider]
+      end
+
       def queue
         class_variable_get(:@@toiler_options)[:queue]
       end
 
+      # kept for compatibility reasons
       def auto_visibility_timeout?
         class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
       end
 
+      def deadline_extension?
+        class_variable_get(:@@toiler_options)[:deadline_extension]
+      end
+
       def auto_delete?
         class_variable_get(:@@toiler_options)[:auto_delete]
       end

data/lib/toiler.rb

@@ -1,4 +1,8 @@
+# frozen_string_literal: true
+
 require 'aws-sdk-sqs'
+require 'google/cloud/pubsub'
+require 'grpc'
 require 'toiler/utils/environment_loader'
 require 'toiler/utils/logging'
 require 'toiler/utils/argument_parser'
@@ -10,13 +14,19 @@ require 'toiler/version'
 module Toiler
   @worker_class_registry = {}
   @options = {
-    aws: {}
+    aws: {},
+    gcp: {}
   }
   @fetchers = {}
   @processor_pools = {}
+  @aws_client = nil
+  @gcp_client = nil
 
   attr_reader :worker_class_registry, :options, :fetchers, :processor_pools
-  module_function :worker_class_registry, :options, :fetchers, :processor_pools
+  attr_accessor :aws_client, :gcp_client
+
+  module_function :worker_class_registry, :options, :fetchers, :processor_pools,
+                  :aws_client, :gcp_client, :aws_client=, :gcp_client=
 
   module_function
 
@@ -63,10 +73,10 @@ module Toiler
   def default_options
     {
       auto_visibility_timeout: false,
+      deadline_extension: false,
       concurrency: 1,
       auto_delete: false,
-      shutdown_timeout: 5,
-      batch: false
+      shutdown_timeout: 5
     }
   end
 

data/spec/models/fetcher_spec.rb

@@ -4,19 +4,45 @@ require 'toiler/actor/fetcher'
 RSpec.describe Toiler::Actor::Fetcher, type: :model do
   let(:queue) { 'default' }
   let(:client) { double(:aws_sqs_client) }
+  let(:aws_queue) { double(:aws_queue) }
+  let(:gcp_queue) { double(:gcp_queue) }
 
   before do
     allow_any_instance_of(Toiler::Actor::Fetcher).to receive(:log).and_return(true)
     allow_any_instance_of(Toiler::Actor::Fetcher).to receive(:tell)
-    allow_any_instance_of(Toiler::Aws::Queue).to receive(:visibility_timeout).and_return(100)
+    allow(Toiler::Aws::Queue).to receive(:new).and_return(aws_queue)
+    allow(Toiler::Gcp::Queue).to receive(:new).and_return(gcp_queue)
+    allow(aws_queue).to receive(:ack_deadline).and_return(100)
+    allow(gcp_queue).to receive(:ack_deadline).and_return(100)
     allow(client).to receive(:get_queue_url).with(queue_name: 'default').and_return double(:queue, queue_url: 'http://aws.fake/queue')
   end
 
   describe "#new" do
-    it 'completes sucessfully' do
-      fetcher = described_class.new(queue, client, 1)
-      expect(fetcher).to have_received(:tell).with(:poll_messages)
-      expect(fetcher.executing?).to eq(false)
+    context 'default' do
+      it 'completes sucessfully' do
+        fetcher = described_class.new(queue, 1, nil)
+        expect(aws_queue).to have_received(:ack_deadline)
+        expect(fetcher).to have_received(:tell).with(:pull_messages)
+        expect(fetcher.executing?).to eq(false)
+      end
+    end
+
+    context 'aws' do
+      it 'completes sucessfully' do
+        fetcher = described_class.new(queue, 1, :aws)
+        expect(aws_queue).to have_received(:ack_deadline)
+        expect(fetcher).to have_received(:tell).with(:pull_messages)
+        expect(fetcher.executing?).to eq(false)
+      end
+    end
+
+    context 'gcp' do
+      it 'completes sucessfully' do
+        fetcher = described_class.new(queue, 1, :gcp)
+        expect(gcp_queue).to have_received(:ack_deadline)
+        expect(fetcher).to have_received(:tell).with(:pull_messages)
+        expect(fetcher.executing?).to eq(false)
+      end
     end
   end
 end

data/spec/models/supervisor_spec.rb

@@ -2,7 +2,6 @@ require 'spec_helper'
 
 require 'toiler/actor/supervisor'
 RSpec.describe Toiler::Actor::Supervisor, type: :model do
-  let(:sqs_client) { double(:client) }
   describe "#new" do
     it 'only spawns fetchers for active workers' do
       class InactiveWorker
@@ -12,8 +11,7 @@ RSpec.describe Toiler::Actor::Supervisor, type: :model do
       end
 
       Toiler.options.merge!(active_queues: ['default'])
-      expect(::Aws::SQS::Client).to receive(:new).and_return(sqs_client)
-      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_default, supervise: true, args: ['default', sqs_client, 1])
+      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_default, supervise: true, args: ['default', 1, nil])
       expect(Concurrent::Actor::Utils::Pool).to receive(:spawn!).with(:processor_pool_default, 1)
       supervisor = described_class.new
     end
@@ -23,5 +21,18 @@ RSpec.describe Toiler::Actor::Supervisor, type: :model do
       expect(Toiler::Utils::Logging.logger).to receive(:warn).with("No worker assigned to queue: missing")
       Toiler.active_worker_class_registry
     end
+
+    it 'sepcifies provider' do
+      class InactiveWorker
+        include Toiler::Worker
+        toiler_options queue: 'gcp_queue', provider: :gcp
+        def perform(sqs_message, body); end
+      end
+
+      Toiler.options.merge!(active_queues: ['gcp_queue'])
+      expect(Toiler::Actor::Fetcher).to receive(:spawn!).with(name: :fetcher_gcp_queue, supervise: true, args: ['gcp_queue', 1, :gcp])
+      expect(Concurrent::Actor::Utils::Pool).to receive(:spawn!).with(:processor_pool_gcp_queue, 1)
+      supervisor = described_class.new
+    end
   end
 end