stapfen 2.2.1 → 3.0.2.21

checksums.yaml

@@ -1,7 +1,15 @@
---- 
-SHA1: 
-  metadata.gz: ca9dc7da001120adcf33f82c7223b68eabf8d6f6
-  data.tar.gz: 3ac1d65ea2159fa5043bb4a33c33da2edf3fb926
-SHA512: 
-  metadata.gz: 0f36e12586128c21c87a2e9d401c2e29b83c42003ac57a4518161c4fce0b0897dfe24b93907423553649265c07f2a9ecf694713797779a5641425d68d6038ad1
-  data.tar.gz: c2fbfe0c65cebf3e8a4a9c6322866cd01513907aa827ee59d31e332697e594036084645cd1bcdd4347ed27982d958c4f97661005f52a04bba19179b7c59036c7
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OGRlODBmYzg5OWFlNmMwYTdjYTc4MDkwY2ZiNzMyYTVhODY2YjgxNA==
+  data.tar.gz: !binary |-
+    NDQ0YTIwZjRlNjdiMzg4NTE3YThiZDZjOGJhOGFjZWI4OWM1ODg3Mw==
+SHA512:
+  metadata.gz: !binary |-
+    MzI4ZDUwNTg4OTA1ZGFkNWY5ZjVjMzU0N2E3NTFkOGI5ZTliYWI0NTliYzkz
+    N2RhNGFjZDg2YjM4NmQ5YTllYjNmMDBiMGQ1MDAwOWFmYzM5NjVmNDk4ZWI3
+    YTBjODk1N2RhMmUyMTMyZmE5MjNiZmI5ODAxZTA4ZDA1YjNjZjM=
+  data.tar.gz: !binary |-
+    MTA3MTg2MTU4NWVhY2Y2YTlkY2QzOGJjOWMwNTVhNDFmNTk4ZTFkMTM3ZTM3
+    OTBkNzk0ODdmNjBjODlkOTM3NzkzMjQwZDRkMzE4OTI0YWJlYTQ3MTEyMDA2
+    ZTFjOWI4YjY5MzE3ZGUzNzgzOWFmMzM2MTlkNGEyMTQwY2Y5YTY=

data/.travis.yml

@@ -1,7 +1,10 @@
 language: ruby
 rvm:
-  - ruby-head
-  - 2.1
-  - 1.9.3
-  - jruby
-
+- ruby-head
+- 2.1
+- 1.9.3
+- jruby
+deploy:
+  provider: rubygems
+  api_key:
+    secure: Nec0EJ9uI21cWjT+e2Yo/MWb03QsQMPk6jMStMcYeTKfNtk/NntPl1LJOQ73JV625QVAItaRzuXDbCfV9Sm2c6zvWJapEJkVp7DAQiZshzKkxSb94N3Uo+Jzgfw0m08Vz6x3af8bLMQgkbcVjLDpbz6L7H2nIf4t1UkJH/I8CeU=

data/Gemfile

@@ -10,7 +10,4 @@ group :development do
   gem 'rake'
   gem 'rspec'
   gem 'rspec-its'
-  gem 'pry'
-  gem 'debugger', :platform => :mri
-  gem 'debugger-pry', :platform => :mri
 end

data/README.md

@@ -20,10 +20,11 @@ Consider the following `myworker.rb` file:
 
 ```ruby
 class MyWorker < Stapfen::Worker
-  use_stomp!
-
-  configure do
-    {
+  configure do |worker|
+    # You can also specify your own logger, but this is the default...
+    worker.logger = Logger.new(STDOUT)
+    worker.protocol = STOMP
+    worker.client_options = {
       :hosts => [
         {
           :host => 'localhost',
@@ -32,18 +33,14 @@ class MyWorker < Stapfen::Worker
           :passcode => 'guest',
           :ssl => false
         }
-      ]
+      ],
+      :topic => 'thequeue',
+      :dead_letter_queue => '/queue/dlq',
+      :max_redeliveries => 0
     }
   end
 
-  # [Optional] Set up a logger for each worker instance
-  log do
-    Logger.new(STDOUT)
-  end
-
-  consume 'thequeue', :dead_letter_queue => '/queue/dlq',
-                      :max_redeliveries => 0 do |message|
-
+  consume do |message|
     data = expensive_computation(message.body)
     # Save my data, or do something worker-specific with it
     persist(data)
@@ -57,13 +54,10 @@ end
 MyWorker.run!
 ```
 
+When using the STOMP protocol, `worker.client_options` can be set with any of the attributes described in a `Stomp::Client` [connection
+hash](https://github.com/stompgem/stomp#hash-login-example-usage-this-is-the-recommended-login-technique) as well as any `subscription` options.
 
-When using the STOMP protocol, the value returned from the `configure` block is expected to be a valid
-`Stomp::Client` [connection
-hash](https://github.com/stompgem/stomp#hash-login-example-usage-this-is-the-recommended-login-technique).
-
-In the case of the JMS protocol, the value returned from the `configure` block
-is expected to be a valid [configuration
+When using the JMS protocol, `worker.client_options` can be set with any of the attributes described in [configuration
 hash](https://github.com/reidmorrison/jruby-jms#consumer) for the
 [jruby-jms](https://github.com/reidmorrison/jruby-jms) gem.
 
@@ -76,30 +70,46 @@ require 'stapfen'
 require 'stapfen/worker'
 
 class MyWorker < Stapfen::Worker
-  use_kafka!
-
-  configure do
-    {
-      :topic => 'test',               # not required
+  configure do |worker|
+    # You can also specify your own logger, but this is the default...
+    worker.logger = Logger.new(STDOUT)
+    worker.protocol = KAFKA
+    worker.client_options = {
+      :topic => 'test',
       :groupId => 'groupId',
       :zookeepers => 'localhost:2181' # comma separated string of zookeepers
     }
   end
 
-  # /topic/test - topic says its a topic, test is the actual topic name
-  consume '/topic/test' do |message|
+  consume do |message|
     puts "Recv: #{message.body}"
   end
 end
 
 MyWorker.run!
 ```
+
+##### Notes
+* Testing with Kafka
+  * Start Staphen worker first
+  * Using producer included with kafka
+    * Produce some messages
+      * ```echo foobar | bin/kafka-console-producer.sh --broker-list <brokers> --topic <topic>```
+    * Worker should be able to read the message
+* using the same groupId a consumer will start reading from the last offset that was read by a consumer from the same group
+  * For example, Given 2 consumers belong to the same groupId
+    * Consumer1 reads a few messages and dies
+    * A producer produces 5 messages
+    * Consumer2 starts up and will receive the 5 messages produced because it started at the last offset of Consumer1
+
 ---
 
 It is also important to note that the `consume` block will be invoked inside an
 **instance** of `MyWorker` and will execute inside its own `Thread`, so take
 care when accessing other shared resources.
 
+Also note you'll need to include the zk gem manually.
+
 ### Fallback and dead-letter-queue support
 
 The consume block accepts the usual subscriptions headers, as well as two
@@ -109,7 +119,6 @@ the block returns `false`; after `:max_redeliveries`, it will send the message
 to `:dead_letter_queue`.  `consume` blocks without these headers will fail
 silently rather than unreceive.
 
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -128,6 +137,7 @@ Or install it yourself as:
 
 Download this from jar from Maven Central
   * [activemq-all-5.8.0.jar](http://search.maven.org/#artifactdetails%7Corg.apache.activemq%7Cactivemq-all%7C5.8.0%7Cjar)
+  * `wget -O activemq-all-5.8.0.jar http://search.maven.org/remotecontent?filepath=org/apache/activemq/activemq-all/5.8.0/activemq-all-5.8.0.jar`
   * Put it in gem root
   * ```rake spec```
 

data/lib/stapfen.rb

@@ -1,3 +1,5 @@
+require 'thread_safe'
+
 begin
   require 'stomp'
 rescue LoadError
@@ -16,10 +18,22 @@ require 'stapfen/client'
 require 'stapfen/worker'
 
 module Stapfen
-  class ConfigurationError < StandardError
+  class ConfigurationError < StandardError; end
+  class ConsumeError < StandardError; end
+  class InvalidMessageError < StandardError; end
+
+  def self.logger=(instance)
+    @logger = instance
   end
-  class ConsumeError < StandardError
+
+  def self.logger
+    @logger ||= default_logger
   end
-  class InvalidMessageError < StandardError
+
+  private
+
+  def self.default_logger
+    require 'logger'
+    Logger.new(STDOUT)
   end
 end

data/lib/stapfen/client/kafka.rb

@@ -20,6 +20,7 @@ module Stapfen
       # @option configuration [String] :topic The kafka topic
       # @option configuration [String] :groupId The kafka groupId
       # @option configuration [String] :zookeepers Comma separated list of zookeepers
+      # @option configuration [Hash]   :consumer_opts Options for Hermann consumer
       #
       # @raises [ConfigurationError] if required configs are not present
       def initialize(configuration)
@@ -28,8 +29,9 @@ module Stapfen
         @topic      = @config[:topic]
         @groupId    = @config[:groupId]
         @zookeepers = @config[:zookeepers]
+        opts        = @config[:consumer_opts]
         raise ConfigurationError unless @groupId && @zookeepers
-        @connection = Hermann::Consumer.new(@topic, @groupId, @zookeepers)
+        @connection = Hermann::Consumer.new(@topic, @groupId, @zookeepers, opts)
       end
 
       # This method is not implemenented
@@ -42,6 +44,14 @@ module Stapfen
         false
       end
 
+      # API compatibilty method, doesn't actually indicate that the connection
+      # is closed. Will only return true if no connection currently exists
+      #
+      # @return [Boolean]
+      def closed?
+        return connection.nil?
+      end
+
       # Closes the consumer threads created by kafka.
       #
       # @return [Boolean] True/false depending on whether we actually closed
@@ -61,8 +71,7 @@ module Stapfen
       #
       # @params [block] block to yield consumed messages
       def subscribe(destination, headers={}, &block)
-        destination = Stapfen::Destination.from_string(destination)
-        connection.consume(destination.as_kafka, &block)
+        connection.consume(destination, &block)
       end
 
       def runloop

data/lib/stapfen/version.rb

@@ -1,3 +1,3 @@
 module Stapfen
-  VERSION = '2.2.1'
+  VERSION = '3.0.2'
 end

data/lib/stapfen/worker.rb

@@ -1,58 +1,145 @@
-require 'stomp'
-require 'stapfen/logger'
 require 'stapfen/destination'
 require 'stapfen/message'
 
 module Stapfen
   class Worker
-    include Stapfen::Logger
+    KAFKA = :kafka.freeze
+    STOMP = :stomp.freeze
+    JMS = :jms.freeze
 
-    # Class variables!
-    @@signals_handled = false
-    @@workers = []
+    attr_accessor :client_options, :protocol, :logger, :stapfen_client
 
     class << self
-      attr_accessor :configuration, :consumers, :logger, :destructor
-    end
+      attr_accessor :instance_configuration, :consumers, :destructor
+
+      def configure(&configuration_block)
+        unless block_given?
+          raise Stapfen::ConfigurationError, "Method `configure` requires a block"
+        end
+        self.instance_configuration = configuration_block
+      end
+
+      # Instantiate a new +Worker+ instance and run it
+      def run!
+        worker = self.new
+
+        @@workers << worker
+
+        handle_signals
+
+        worker.run
+      end
+
+      # Main message consumption block
+      def consume(config_overrides={}, &consume_block)
+        unless block_given?
+          raise Stapfen::ConsumeError, "Method `consume` requires a block"
+        end
+        @consumers ||= ThreadSafe::Array.new
+        @consumers << [config_overrides, consume_block]
+      end
+
+      # Optional method, specifes a block to execute when the worker is shutting
+      # down.
+      def shutdown(&block)
+        @destructor = block
+      end
+
+      # Return all the currently running Stapfen::Worker instances in this
+      # process
+      def workers
+        @@workers
+      end
+
+      # Invoke +exit_cleanly+ on each of the registered Worker instances that
+      # this class is keeping track of
+      #
+      # @return [Boolean] Whether or not we've exited/terminated cleanly
+      def exit_cleanly
+        return false if workers.empty?
+
+        cleanly = true
+        workers.each do |w|
+          begin
+            w.exit_cleanly
+          rescue StandardError => ex
+            $stderr.write("Failure while exiting cleanly #{ex.inspect}\n#{ex.backtrace}")
+            cleanly = false
+          end
+        end
 
-    # Instantiate a new +Worker+ instance and run it
-    def self.run!
-      worker = self.new
+        if RUBY_PLATFORM == 'java'
+          Stapfen.logger.info 'Telling the JVM to exit cleanly'
+          Java::JavaLang::System.exit(0)
+        end
 
-      @@workers << worker
+        return cleanly
+      end
 
-      handle_signals
+      # Utility method to set up the proper worker signal handlers
+      def handle_signals
+        return if @@signals_handled
+
+        Signal.trap(:INT) do
+          self.exit_cleanly
+          exit!
+        end
+
+        Signal.trap(:TERM) do
+          self.exit_cleanly
+        end
+
+        @@signals_handled = true
+      end
+
+      # Class variables are put in this method to allow for "reset" style
+      # functionality if needed. Useful for testing (see worker_spec.rb).
+      def set_class_variable_defaults
+        @@signals_handled = false
+        @@workers = ThreadSafe::Array.new
+      end
 
-      worker.run
     end
 
-    # Expects a block to be passed which will yield the appropriate
-    # configuration for the Stomp gem. Whatever the block yields will be passed
-    # directly into the {{Stomp::Client#new}} method
-    def self.configure(&block)
-      unless block_given?
-        raise Stapfen::ConfigurationError
+    set_class_variable_defaults
+
+    ############################################################################
+    # Instance Methods
+    ############################################################################
+
+    def initialize
+      instance_configuration = self.class.instance_configuration
+      if instance_configuration
+        self.configure &instance_configuration
       end
-      @configuration = block
+      self.client_options ||= {}
+    end
+
+    def configure(&configuration_block)
+      self.instance_eval &configuration_block
+    end
+
+    def logger
+      @logger ||= Stapfen.logger
     end
 
     # Force the worker to use STOMP as the messaging protocol (default)
     #
     # @return [Boolean]
-    def self.use_stomp!
+    def use_stomp!
       begin
         require 'stomp'
       rescue LoadError
-        puts "You need the `stomp` gem to be installed to use stomp!"
+        Stapfen.logger.info 'You need the `stomp` gem to be installed to use stomp!'
         raise
       end
 
-      @protocol = 'stomp'
+      @protocol = STOMP
       return true
     end
 
-    def self.stomp?
-      @protocol.nil? || @protocol == 'stomp'
+    def stomp?
+      @protocol == STOMP
     end
 
     # Force the worker to use JMS as the messaging protocol.
@@ -60,25 +147,25 @@ module Stapfen
     # *Note:* Only works under JRuby
     #
     # @return [Boolean]
-    def self.use_jms!
+    def use_jms!
       unless RUBY_PLATFORM == 'java'
-        raise Stapfen::ConfigurationError, "You cannot use JMS unless you're running under JRuby!"
+        raise Stapfen::ConfigurationError, 'You cannot use JMS unless you are running under JRuby!'
       end
 
       begin
         require 'java'
         require 'jms'
       rescue LoadError
-        puts "You need the `jms` gem to be installed to use JMS!"
+        Stapfen.logger.info 'You need the `jms` gem to be installed to use JMS!'
         raise
       end
 
-      @protocol = 'jms'
+      @protocol = JMS
       return true
     end
 
-    def self.jms?
-      @protocol == 'jms'
+    def jms?
+      @protocol == JMS
     end
 
     # Force the worker to use Kafka as the messaging protocol.
@@ -86,158 +173,88 @@ module Stapfen
     # *Note:* Only works under JRuby
     #
     # @return [Boolean]
-    def self.use_kafka!
+    def use_kafka!
       unless RUBY_PLATFORM == 'java'
-        raise Stapfen::ConfigurationError, "You cannot use Kafka unless you're running under JRuby!"
+        raise Stapfen::ConfigurationError, 'You cannot use Kafka unless you are running under JRuby!'
       end
 
       begin
         require 'java'
         require 'hermann'
       rescue LoadError
-        puts "You need the `hermann` gem to be installed to use Kafka!"
+        Stapfen.logger.info 'You need the `hermann` gem to be installed to use Kafka!'
         raise
       end
 
-      @protocol = 'kafka'
+      @protocol = KAFKA
       return true
     end
 
-    def self.kafka?
-      @protocol == 'kafka'
-    end
-
-    # Optional method, should be passed a block which will yield a {{Logger}}
-    # instance for the Stapfen worker to use
-    def self.log(&block)
-      @logger = block
-    end
-
-    # Main message consumption block
-    def self.consume(queue_name, headers={}, &block)
-      unless block_given?
-        raise Stapfen::ConsumeError, "Cannot consume #{queue_name} without a block!"
-      end
-      @consumers ||= []
-      @consumers << [queue_name, headers, block]
-    end
-
-    # Optional method, specifes a block to execute when the worker is shutting
-    # down.
-    def self.shutdown(&block)
-      @destructor = block
-    end
-
-    # Return all the currently running Stapfen::Worker instances in this
-    # process
-    def self.workers
-      @@workers
-    end
-
-    # Invoke +exit_cleanly+ on each of the registered Worker instances that
-    # this class is keeping track of
-    #
-    # @return [Boolean] Whether or not we've exited/terminated cleanly
-    def self.exit_cleanly
-      return false if workers.empty?
-
-      cleanly = true
-      workers.each do |w|
-        begin
-          w.exit_cleanly
-        rescue StandardError => ex
-          $stderr.write("Failure while exiting cleanly #{ex.inspect}\n#{ex.backtrace}")
-          cleanly = false
-        end
-      end
-
-      if RUBY_PLATFORM == 'java'
-        info "Telling the JVM to exit cleanly"
-        Java::JavaLang::System.exit(0)
-      end
-
-      return cleanly
+    def kafka?
+      @protocol == KAFKA
     end
 
-    # Utility method to set up the proper worker signal handlers
-    def self.handle_signals
-      return if @@signals_handled
-
-      Signal.trap(:INT) do
-        self.exit_cleanly
-        exit!
-      end
-
-      Signal.trap(:TERM) do
-        self.exit_cleanly
-      end
-
-      @@signals_handled = true
+    def protocol
+      @protocol ||= KAFKA
     end
 
-
-
-    ############################################################################
-    # Instance Methods
-    ############################################################################
-
-    attr_accessor :client
-
     def run
-      if self.class.stomp?
+      case protocol
+      when STOMP
         require 'stapfen/client/stomp'
-        @client = Stapfen::Client::Stomp.new(self.class.configuration.call)
-      elsif self.class.jms?
+        stapfen_client = Stapfen::Client::Stomp.new(client_options)
+      when JMS
         require 'stapfen/client/jms'
-        @client = Stapfen::Client::JMS.new(self.class.configuration.call)
-      elsif self.class.kafka?
+        stapfen_client = Stapfen::Client::JMS.new(client_options)
+      when KAFKA
         require 'stapfen/client/kafka'
-        @client = Stapfen::Client::Kafka.new(self.class.configuration.call)
+        stapfen_client = Stapfen::Client::Kafka.new(client_options)
+      else
+        raise 'No client specified'
       end
 
-      debug("Running with #{@client} inside of Thread:#{Thread.current.inspect}")
+      logger.info("Running with #{stapfen_client} inside of Thread:#{Thread.current.inspect}")
 
-      @client.connect
+      stapfen_client.connect
 
-      self.class.consumers.each do |name, headers, block|
-        unreceive_headers = {}
-        [:max_redeliveries, :dead_letter_queue].each do |sym|
-          unreceive_headers[sym] = headers[sym] if headers.has_key? sym
-        end
+      self.class.consumers.each do |config_overrides, block|
+        consumer_config = client_options.merge(config_overrides)
+        consumer_topic = consumer_config[:topic]
+        consumer_can_unreceive = !(consumer_config.keys & [:max_redeliveries, :dead_letter_queue]).empty?
 
         # We're taking each block and turning it into a method so that we can
         # use the instance scope instead of the blocks originally bound scope
         # which would be at a class level
-        method_name = name.gsub(/[.|\-]/, '_').to_sym
-        self.class.send(:define_method, method_name, &block)
+        methodized_topic = consumer_topic.gsub(/[.|\-]/, '_').to_sym
+        self.class.send(:define_method, methodized_topic, &block)
 
-        client.subscribe(name, headers) do |m|
-          message = nil
-          if self.class.stomp?
-            message = Stapfen::Message.from_stomp(m)
+        stapfen_client.subscribe(consumer_topic, consumer_config) do |message_entity|
+          stapfen_message = nil
+          if stomp?
+            stapfen_message = Stapfen::Message.from_stomp(message_entity)
           end
 
-          if self.class.jms?
-            message = Stapfen::Message.from_jms(m)
+          if jms?
+            stapfen_message = Stapfen::Message.from_jms(message_entity)
           end
 
-          if self.class.kafka?
-            message = Stapfen::Message.from_kafka(m)
+          if kafka?
+            stapfen_message = Stapfen::Message.from_kafka(message_entity)
           end
 
-          success = self.send(method_name, message)
+          success = self.send(methodized_topic, stapfen_message)
 
           unless success
-            if client.can_unreceive? && !unreceive_headers.empty?
-              client.unreceive(m, unreceive_headers)
+            if stapfen_client.can_unreceive? && consumer_can_unreceive
+              stapfen_client.unreceive(message_entity, consumer_config)
             end
           end
         end
       end
 
       begin
-        client.runloop
-        warn("Exiting the runloop for #{self}")
+        stapfen_client.runloop
+        logger.info("Exiting the runloop for #{self}")
       rescue Interrupt
         exit_cleanly
       end
@@ -246,19 +263,19 @@ module Stapfen
     # Invokes the shutdown block if it has been created, and closes the
     # {{Stomp::Client}} connection unless it has already been shut down
     def exit_cleanly
-      info("#{self} exiting ")
+      logger.info("#{self} exiting ")
       self.class.destructor.call if self.class.destructor
 
-      info "Killing client"
+      logger.info 'Killing client'
       begin
         # Only close the client if we have one sitting around
-        if client
-          unless client.closed?
-            client.close
+        if stapfen_client
+          unless stapfen_client.closed?
+            stapfen_client.close
           end
         end
       rescue StandardError => exc
-        error "Exception received while trying to close client! #{exc.inspect}"
+        logger.error "Exception received while trying to close client! #{exc.inspect}"
       end
     end
   end