skein 0.3.7 → 0.8.1

data/lib/skein/client/subscriber.rb

@@ -19,15 +19,38 @@ class Skein::Client::Subscriber < Skein::Connected
   def listen(block = true)
     case (@subscribe_queue.class.to_s.split(/::/)[0])
     when 'Bunny'
-      @subscribe_queue.subscribe(block: block) do |delivery_info, properties, payload|
-        yield(JSON.load(payload), delivery_info, properties)
+      begin
+        @subscribe_queue.subscribe(block: block) do |delivery_info, properties, payload|
+          yield(JSON.load(payload), delivery_info, properties)
+        end
       end
     when 'MarchHare'
-      @subscribe_queue.subscribe(block: block) do |metadata, payload|
-        yield(JSON.load(payload), metadata)
+      begin
+        @subscribe_queue.subscribe(block: block) do |metadata, payload|
+          yield(JSON.load(payload), metadata)
+        end
+      rescue MarchHare::ChannelAlreadyClosed
+        # Connection got killed outside of thread, so shut-down and move on
       end
     else
       raise "Unknown queue type #{@subscribe_queue.class}, cannot listen."
     end
   end
+
+  def close(delete_queue: false)
+    if (delete_queue)
+      begin
+        @subscribe_queue.delete
+      rescue => e
+        case (e.class.to_s)
+        when 'Bunny::ChannelAlreadyClosed', 'MarchHare::ChannelAlreadyClosed'
+          # Tried to delete, but this has already been shut down
+        else
+          raise e
+        end
+      end
+    end
+
+    super()
+  end
 end

data/lib/skein/client/worker.rb

@@ -1,97 +1,121 @@
 require 'json'
 
 class Skein::Client::Worker < Skein::Connected
+  # == Properties ===========================================================
+
+  attr_reader :operations
+  attr_reader :queue_name
+
+  # == Exceptions ===========================================================
+
+  class RejectMessage < Exception
+  end
+
+  class RetryMessage < Exception
+  end
+
+  # == Class Methods ========================================================
+
   # == Instance Methods =====================================================
 
-  def initialize(queue_name, exchange_name: nil, connection: nil, context: nil, concurrency: nil)
-    super(connection: connection, context: context)
+  def initialize(queue_name, exchange_name: nil, connection: nil, context: nil, concurrency: nil, durable: nil, auto_delete: false, routing_key: nil, ident: nil)
+    super(connection: connection, context: context, ident: ident)
 
-    lock do
-      @reply_exchange = self.channel.default_exchange
-      @queue = self.channel.queue(queue_name, durable: !!queue_name.match(/\S/))
+    @exchange_name = exchange_name
+    @queue_name = queue_name.dup.freeze
+    @routing_key = routing_key
+    @durable = durable.nil? ? !!@queue_name.match(/\S/) : !!durable
+    @operations = [ ]
+    @auto_delete = auto_delete
 
-      if (exchange_name)
-        @exchange = self.channel.direct(exchange_name, durable: true)
+    concurrency &&= concurrency.to_i
+    concurrency ||= 1
 
-        @queue.bind(@exchange)
+    concurrency.times do |i|
+      with_channel_in_thread(name: 'worker-%d' % i) do |channel, meta|
+        self.establish_subscriber!(channel, meta)
       end
+    end
 
-      @handler = Skein::Handler.for(self)
-      @received = Queue.new
-      @replies = Queue.new
-      @concurrency = concurrency && concurrency.to_i || 1
-      @threads = [ ]
+    self.after_initialize rescue nil
+  end
 
-      @threads << Thread.new do
-        Thread.abort_on_exception = true
+  # Define in derived classes to implement any desired customization to be
+  # performed after initialization.
+  def after_initialize
+  end
 
-        Skein::Adapter.subscribe(@queue) do |payload, delivery_tag, reply_to|
-          @received << [ payload, delivery_tag, reply_to ]
-        end
-      end
+  # Define in derived classes. Willl be called immediately after a request is
+  # received but before any processing occurs.
+  def before_request
+  end
 
-      @threads << Thread.new do
-        Thread.abort_on_exception = true
+  # Define in derived classes. Will be called immediately prior to executing
+  # the worker method.
+  def before_execution(method_name)
+  end
 
-        loop do
-          payload, delivery_tag, reply_to, reply_json = @replies.pop
+  # Define in derived classes. Will be called immediately after executing the
+  # worker method.
+  def after_execution(method_name)
+  end
 
-          channel.acknowledge(delivery_tag, true)
+  # Define in derived classes. Will be called immediately after handling an
+  # RPC call even if an error has occured.
+  def after_request
+  end
 
-          if (reply_to)
-            @reply_exchange.publish(
-              reply_json,
-              routing_key: reply_to,
-              content_type: 'application/json'
-            )
-          end
-        end
-      end
+  # Declared in derived classes. Will be called immediately after an
+  # exception has occurred when processing a request. Any excepions generated
+  # in this method call are suppressed and ignored to avoid being caught
+  # in a loop.
+  def after_exception(e)
+  end
 
-      @concurrency.times do
-        @threads << Thread.new do
-          Thread.abort_on_exception = true
+  # Declared in derived classes. Will be called immediately before the worker
+  # is closed down.
+  def before_close
+  end
 
-          loop do
-            payload, delivery_tag, reply_to = @received.pop
-            thread = Thread.current
+  # Declared in derived classes. Will be called immediately after the worker
+  # has been closed down.
+  def after_close
+  end
 
-            @handler.handle(payload) do |reply_json|
-              @replies << [ payload, delivery_tag, reply_to, reply_json ]
+  def close(delete_queue: false)
+    self.before_close
 
-              if (thread == Thread.current)
-                thread = nil
-              else
-                thread.wakeup
-              end
-            end
+    @operations.each do |meta|
+      subscriber = meta[:subscriber]
 
-            thread and Thread.stop
-          end
-        end
+      if (subscriber.respond_to?(:gracefully_shut_down))
+        subscriber.gracefully_shut_down
       end
+
+      thread = meta[:thread]
+
+      thread.respond_to?(:terminate!) ? thread.terminate! : thread.kill
+      thread.join
     end
 
-    self.after_initialize
-  end
+    if (delete_queue)
+      # The connection may have been terminated, so reconnect and delete
+      # the queue if necessary.
+      channel = @connection.create_channel
 
-  # Extend this in derived classes to implement any desired customization to
-  # be performed after initialization
-  def after_initialize
-  end
+      channel.queue(@queue_name, durable: @durable).delete
 
-  def close
-    @threads.each do |thread|
-      thread.kill
-      thread.join
+      channel.close
     end
 
-    super
+    super()
+
+    self.after_close
   end
 
   def join
-    @threads.each do |thread|
-      thread.join
+    @operations.each do |meta|
+      meta[:thread].join
     end
   end
 
@@ -100,4 +124,138 @@ class Skein::Client::Worker < Skein::Connected
     # callback-style delegation.
     false
   end
+
+  # Signal that the current operation should be abandoned and not retried.
+  def reject!
+    raise RejectMessage
+  end
+
+  # Signal that the current operation should be abandoned and retried later.
+  def retry!
+    raise RetryMessage
+  end
+
+protected
+  def state_tracker
+    {
+      method: nil,
+      started: nil,
+      finished: nil
+    }
+  end
+
+  def metrics_tracker
+    Hash.new(0).merge(
+      time: 0.0,
+      errors: Hash.new(0)
+    )
+  end
+
+  def in_thread
+    @operations << {
+      thread: Thread.new do
+        Thread.abort_on_exception = true
+
+        yield
+      end
+    }
+  end
+
+  def establish_queue!(channel)
+    queue = channel.queue(
+      @queue_name,
+      durable: @durable,
+      auto_delete: @auto_delete
+    )
+
+    if (@exchange_name&.match(/\S/))
+      exchange = channel.direct(@exchange_name, durable: true)
+
+      queue.bind(exchange, routing_key: @routing_key || @queue_name)
+    end
+
+    queue
+  end
+
+  def establish_subscriber!(channel, meta)
+    queue = self.establish_queue!(channel)
+
+    meta[:subscriber] = Skein::Adapter.subscribe(queue) do |payload, delivery_tag, reply_to|
+      if (ENV['SKEIN_DEBUG_JSON'] and reply_to)
+        $stdout.puts('%s -> %s' % [ reply_to, payload ])
+      end
+
+      self.context.trap do
+        self.before_request rescue nil
+
+        handler.handle(payload, meta[:metrics], meta[:state]) do |reply_json|
+          if (ENV['SKEIN_DEBUG_JSON'] and reply_to)
+            $stdout.puts('%s <- %s' % [ reply_to, reply_json ])
+          end
+
+          # Secondary (inner) trap required since some handlers are async
+          self.context.trap do
+            # NOTE: begin...end necessary for rescue in Ruby versions below 2.4
+            begin
+              channel.acknowledge(delivery_tag, true)
+
+              if (reply_to)
+                channel.default_exchange.publish(
+                  reply_json,
+                  routing_key: reply_to,
+                  content_type: 'application/json'
+                )
+              end
+
+            rescue RejectMessage
+              # Reject the message
+              channel.reject(delivery_tag, false)
+            rescue RetryMessage
+              # Reject and requeue the message
+              channel.reject(delivery_tag, true)
+            rescue => e
+              self.after_exception(e) rescue nil
+              raise e
+            ensure
+              self.after_request rescue nil
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def with_channel_in_thread(recover: true, name: nil)
+    meta = {
+      metrics: metrics_tracker,
+      state: state_tracker
+    }
+
+    @operations << meta
+
+    meta[:thread] = Thread.new do
+      Thread.abort_on_exception = true
+      Thread.current.name = name
+
+      begin
+        channel = meta[:channel] = self.create_channel
+
+        yield(channel, meta)
+
+        channel.close rescue nil
+
+        redo if (recover)
+
+      ensure
+        # NOTE: The `.close` call may fail for a variety of reasons, but the
+        #       important thing here is an attempt is made, regardless of
+        #       outcome.
+        channel&.close rescue nil
+      end
+    end
+  end
+
+  def handler
+    @handler ||= Skein::Handler.for(self)
+  end
 end

data/lib/skein/client.rb

@@ -24,38 +24,47 @@ class Skein::Client < Skein::Connected
 
   # == Instance Methods =====================================================
 
-  def initialize(connection: nil, context: nil)
-    super(connection: connection, context: context)
+  def initialize(config: nil, connection: nil, context: nil)
+    super(config: config, connection: connection, context: context)
   end
 
-  def rpc(exchange_name = nil, routing_key: nil)
+  def rpc(exchange_name = nil, routing_key: nil, ident: nil, expiration: nil, persistent: nil, durable: nil, timeout: nil)
     Skein::Client::RPC.new(
       exchange_name,
       routing_key: routing_key,
       connection: self.connection,
-      context: self.context
+      context: self.context,
+      ident: ident,
+      expiration: expiration,
+      persistent: persistent,
+      durable: durable,
+      timeout: timeout
     )
   end
 
-  def worker(queue_name)
-    Skein::Client::Worker.new(
+  def worker(queue_name, type = nil, ident: nil, durable: nil)
+    (type || Skein::Client::Worker).new(
       queue_name,
       connection: self.connection,
-      context: self.context
+      context: self.context,
+      ident: ident,
+      durable: durable
     )
   end
 
-  def publisher(queue_name)
+  def publisher(exchange_name, type: nil, durable: nil)
     Skein::Client::Publisher.new(
-      queue_name,
+      exchange_name,
+      type: type,
+      durable: durable,
       connection: self.connection,
       context: self.context
     )
   end
 
-  def subscriber(queue_name, routing_key = nil)
+  def subscriber(exchange_name, routing_key = nil)
     Skein::Client::Subscriber.new(
-      queue_name,
+      exchange_name,
       routing_key,
       connection: self.connection,
       context: self.context

data/lib/skein/config.rb

@@ -87,7 +87,9 @@ class Skein::Config < OpenStruct
     end
 
     if (config_path and File.exist?(config_path))
-      super(DEFAULTS.merge(YAML.load_file(config_path)[self.class.env] || { }))
+      super(DEFAULTS.merge(
+        YAML.load_file(config_path, aliases: true)[self.class.env] || { }
+      ))
     else
       super(DEFAULTS)
     end

data/lib/skein/connected.rb

@@ -4,19 +4,25 @@ class Skein::Connected
   attr_reader :context
   attr_reader :ident
   attr_reader :connection
-  attr_reader :channel
 
   # == Instance Methods =====================================================
 
-  def initialize(connection: nil, context: nil)
+  def initialize(config: nil, connection: nil, context: nil, ident: nil)
     @mutex = Mutex.new
-    @shared_connection = !!connection
 
-    @connection = connection || Skein::RabbitMQ.connect
-    @channel = @connection.create_channel
+    @config = config
+    @connection_shared = !connection
+    @connection = connection
+
+    self.connect
+    @channels = [ ]
 
     @context = context || Skein::Context.new
-    @ident = @context.ident(self)
+    @ident = ident || @context.ident(self)
+  end
+
+  def connection_shared?
+    @connection_shared
   end
 
   def lock
@@ -25,26 +31,91 @@ class Skein::Connected
     end
   end
 
+  def repeat_until_not_nil(delay: 1.0)
+    r = nil
+
+    loop do
+      r = yield
+
+      break if r
+
+      sleep(delay)
+    end
+
+    r
+  end
+
+  def connect
+    @connection ||= repeat_until_not_nil do
+      @connection_shared = false
+      Skein::RabbitMQ.connect(@config)
+    end
+  end
+
+  def reconnect
+    @connection = nil
+
+    self.connect
+  end
+
+  def create_channel(auto_retry: false)
+    channel = begin
+      @connection.create_channel
+
+    rescue RuntimeError
+      sleep(1)
+
+      self.reconnect
+
+      retry
+    end
+
+    if (channel.respond_to?(:prefetch=))
+      channel.prefetch = 1
+    else
+      channel.prefetch(1)
+    end
+
+    @channels << channel
+
+    channel
+  end
+
+  def channel
+    @channel ||= self.create_channel
+  end
+
   def close
     lock do
-      begin
-        @channel and @channel.close
-
-      rescue => e
-        if (defined?(MarchHare))
-          case e
-          when MarchHare::ChannelLevelException, MarchHare::ChannelAlreadyClosed
-            # Ignored since we're finished with the channel anyway
+      @channels.each do |channel|
+        begin
+          channel.close
+
+        rescue => e
+          if (defined?(MarchHare))
+            case (e)
+            when MarchHare::ChannelLevelException, MarchHare::ChannelAlreadyClosed
+              # Ignored since we're finished with the channel anyway
+            else
+              raise e
+            end
+          elsif (defined?(Bunny))
+            case (e)
+            when Bunny::ChannelAlreadyClosed
+              # Ignored since we're finished with the channel anyway
+            else
+              raise e
+            end
           else
             raise e
           end
         end
       end
 
-      @channel = nil
+      @channels = [ ]
 
-      unless (@shared_connection)
-        @connection and @connection.close
+      unless (@connection_shared)
+        @connection&.close
         @connection = nil
       end
     end