emque-consuming 1.0.0.beta4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 40887d498ed3a5b0237e45e3356b28c8b5b4d2df
+  data.tar.gz: 6d19d4ff37dbaf233dfe696e7bf03a5650a5a986
+SHA512:
+  metadata.gz: 5de0803235bb52e85a39d61a944cb0ce60df327f1f9b51a855214cadc2041c5c4bb7825126cb95a01619a0911b865de34b820aa26ec13a640d12100bbe9de1d1
+  data.tar.gz: b6e8634ba543e1b9f3bcd9b859219b601671413a16341beb2a35d184909c4125122cccca842ce24554a8d0614ea2910cff251975bcd9eade678ff371446721d4

data/.gitignore

@@ -0,0 +1,40 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+kafka_*/
+/log/*
+/logs/*
+log-cleaner.log
+Gemfile.lock
+*.log
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+rvm:
+  - 2.1.6
+  - 2.2.2
+  - rbx-2
+script: bundle exec rake
+notifications:
+  recipients:
+    - shane@teamsnap.com
+    - dan@sensiblesitters.com

data/CHANGELOG.md

@@ -0,0 +1,11 @@
+## 1.0.0.beta4 — (unreleased)
+
+### BREAKING CHANGE - New Queue Names
+Applications updating to this version will have new queue names in RabbitMQ.
+After starting up, messages will need to be manually moved
+from the old queue to the new one.
+
+### Failed Message Routing
+Messages that are not acknolwedged due to a consumer error will now be routed
+into a `service_name.error` queue. These can then be inspected and be discarded,
+purged, or manually moved back to the primary queue for re-processing.

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+gem "simplecov", :require => false
+gem "coveralls", :require => false
+gem "pry", :require => false
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 TeamSnap
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,205 @@
+[![Build Status](https://travis-ci.org/emque/emque-consuming.png)](https://travis-ci.org/emque/emque-consuming)
+
+# Emque::Consuming
+
+Emque Consuming is a Ruby application framework that includes everything needed
+to create and run application capable of consuming messages from a message
+broker in a Pub/sub architecture. Messages can be produced with the
+[emque-producing](https://github.com/teamsnap/emque-producing) library.
+
+## Adapters
+
+We currently only support RabbitMQ. If you would like to add your own adapter,
+take a look at [the adapters directory](https://github.com/teamsnap/emque-consuming/tree/socket-control/lib/emque/consuming/adapters).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "emque-consuming", :git => "https://github.com/teamsnap/emque-consuming"
+# make sure you have bunny for rabbitmq unless you're using a custom adapter
+gem "bunny", "~> 1.7"
+
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install emque-consuming
+
+## Setup
+
+### Easy
+
+Using the `new` generator is the easiest way to get up and running quickly.
+
+```
+emque <options> new name
+```
+
+This command will create a directory "name" with barebones directories and files
+you'll need to get started. You can also use the following command line options
+to customize your application's configuration. Options will be added to the
+config/application.rb file generated.
+
+```
+emque <options> (start|stop|new|console|help) <name (new only)>
+    # ...
+    -S, --socket PATH                PATH to the application's unix socket
+    -b, --bind IP:PORT               IP & port for the http status application to listen on.
+    # ...
+    -e, --error-limit N              Set the max errors before application suicide
+    -s, --status                     Run the http status application
+    -x, --error-expiration SECONDS   Expire errors after SECONDS
+        --app-name NAME              Run the application as NAME
+    # ...
+```
+
+### Custom
+
+Configure Emque::Consuming in your config/application.rb file. Here is an example:
+
+```ruby
+# config/application.rb
+require "emque/consuming"
+
+module Example
+  class Application
+    include Emque::Consuming::Application
+
+    initialize_core!
+
+    config.set_adapter(:rabbit_mq, :url => ENV["RABBITMQ_URL"], :prefetch => 10)
+
+    # customize the error thresholds
+    # config.error_limit = 10
+    # config.error_expiration = 300
+
+    # enable the status application
+    # config.status = :on
+
+    # errors will be logged, but if more is needed, that can be added
+    # config.error_handlers << Proc.new {|ex, context|
+    #  send an email, send to HoneyBadger, etc
+    # }
+
+    # do something when shutdown occurs
+    # config.shutdown_handlers << Proc.new { |context|
+    #   notify slack, pagerduty or send an email
+    # }
+  end
+end
+```
+
+You'll also want to set up a routes.rb file:
+
+```ruby
+# config/routes.rb
+
+Example::Application.router.map do
+  topic "events" => EventsConsumer do
+    map "events.new" => "new_event"
+    # ...
+  end
+
+  # ...
+end
+```
+
+and a consumer for each topic:
+
+```ruby
+# app/consumers/events_consumer.rb
+
+class EventsConsumer
+  include Emque::Consuming.consumer
+
+  def new_event(message)
+    # NOTE: message is an immutable [Virtus](https://github.com/solnic/virtus) Value Object.
+    # Check it out here: https://github.com/teamsnap/emque-consuming/blob/master/lib/emque/consuming/message.rb
+
+    # You don't have to use (pipe)[https://github.com/teamsnap/emque-consuming/blob/master/lib/emque/consuming/consumer/common.rb#L23], be we love it!
+    pipe(message, :through => [
+      :shared_action, :do_something_with_new_event
+    ])
+  end
+
+  private
+
+  def shared_action(message)
+    # ...
+  end
+
+  def do_something_with_new_event(message)
+    # ...
+  end
+end
+```
+
+## Usage
+
+Emque::Consuming provides a command line interface:
+
+```
+$ bundle exec emque help
+
+emque <options> (start|stop|new|console|help) <name (new only)>
+    -P, --pidfile PATH               Store pid in PATH
+    -S, --socket PATH                PATH to the application's unix socket
+    -b, --bind IP:PORT               IP & port for the http status application to listen on.
+    -d, --daemon                     Daemonize the application
+    -e, --error-limit N              Set the max errors before application suicide
+    -s, --status                     Run the http status application
+    -x, --error-expiration SECONDS   Expire errors after SECONDS
+        --app-name NAME              Run the application as NAME
+        --env (ex. production)       Set the application environment, overrides EMQUE_ENV
+```
+
+and a series of rake commands:
+
+```
+$ bundle exec rake -T
+
+rake emque:configuration        # Show the current configuration of a running instance (accepts SOCKET)
+rake emque:console              # Start a pry console
+rake emque:errors:clear         # Clear all outstanding errors (accepts SOCKET)
+rake emque:errors:expire_after  # Change the number of seconds to SECONDS before future errors expire (accepts SOCKET)
+rake emque:errors:limit:down    # Decrease the error limit (accepts SOCKET)
+rake emque:errors:limit:up      # Increase the error limit (accepts SOCKET)
+rake emque:restart              # Restart the workers inside a running instance (does not reload code; accepts SOCKET)
+rake emque:routes               # Show the available routes
+rake emque:start                # Start a new instance (accepts PIDFILE, DAEMON)
+rake emque:status               # Show the current status of a running instance (accepts SOCKET)
+rake emque:stop                 # Stop a running instance (accepts SOCKET)
+```
+
+To use the rake commands, add the following lines to your application's Rakefile:
+
+```ruby
+require_relative "config/application"
+require "emque/consuming/tasks"
+```
+
+## Tests
+
+Testing is a bit sparse at the moment, but we're working on it.
+
+To run tests...
+
+```
+bundle exec rspec
+```
+
+## Contributing
+
+FIRST: Read our style guides at https://github.com/teamsnap/guides/tree/master/ruby
+
+1. Fork it ( http://github.com/teamsnap/emque-consuming/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,14 @@
+require "rubygems"
+require "bundler/gem_tasks"
+require "rake/clean"
+require "rspec/core/rake_task"
+require "coveralls/rake/task"
+
+CLOBBER.include("coverage")
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.fail_on_error = false
+end
+task :default => :spec
+
+Coveralls::RakeTask.new

data/bin/emque

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "emque-consuming"
+
+Emque::Consuming::Cli.new(ARGV)

data/emque-consuming.gemspec

@@ -0,0 +1,36 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "emque/consuming/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "emque-consuming"
+  spec.version       = Emque::Consuming::VERSION
+  spec.authors       = ["Ryan Williams", "Dan Matthews"]
+  spec.email         = ["oss@teamsnap.com"]
+  spec.summary       = %q{Microservices framework for Ruby}
+  spec.summary       = %q{Microservices framework for Ruby}
+  spec.homepage      = "https://github.com/teamsnap/emque-consuming"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  spec.required_ruby_version = "~> 2.1.2"
+
+  spec.add_dependency "celluloid", "0.16.0"
+  spec.add_dependency "dante",     "~> 0.2.0"
+  spec.add_dependency "oj",        "~> 2.11.4"
+  spec.add_dependency "virtus",    "~> 1.0"
+  spec.add_dependency "puma",      "~> 2.11.1"
+  spec.add_dependency "pipe-ruby", "~> 0.2.0"
+  spec.add_dependency "inflecto",  "~> 0.0.2"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake",    "~> 10.4.2"
+  spec.add_development_dependency "rspec",   "~> 3.3"
+  spec.add_development_dependency "bunny",   "~> 1.7"
+  spec.add_development_dependency "timecop", "~> 0.7.1"
+  spec.add_development_dependency "daemon_controller", "~> 1.2.0"
+end

data/lib/emque/consuming/actor.rb

@@ -0,0 +1,21 @@
+module Emque
+  module Consuming
+    module Actor
+      def self.included(descendant)
+        descendant.class_eval do
+          include Celluloid
+          include Emque::Consuming::Helpers
+          attr_accessor :shutdown
+          private :shutdown=
+          private :shutdown
+        end
+      end
+
+      def stop(&block)
+        self.shutdown = true
+        block.call if block_given?
+        terminate
+      end
+    end
+  end
+end

data/lib/emque/consuming/adapter.rb

@@ -0,0 +1,47 @@
+require "inflecto"
+
+module Emque
+  module Consuming
+    class AdapterConfigurationError < StandardError; end
+
+    module Adapters; end
+
+    class Adapter
+      extend Forwardable
+
+      attr_reader :name, :options
+
+      def_delegators :namespace, :default_options, :manager
+
+      def initialize(name, opts = {})
+        self.name = name
+        fetch_and_load
+        self.options = default_options.merge(opts)
+      end
+
+      private
+
+      attr_writer :name, :options
+      attr_accessor :namespace
+
+      def fetch_and_load
+        klass = Inflecto.camelize(name.to_s)
+
+        unless Emque::Consuming::Adapters.const_defined?(klass)
+          require "emque/consuming/adapters/#{name}"
+        end
+
+        self.namespace = Inflecto.constantize(
+          "Emque::Consuming::Adapters::#{klass}"
+        )
+
+        namespace.load
+      rescue LoadError
+        raise AdapterConfigurationError, "Unable to load requested adapter"
+      rescue NameError => e
+        $stdout.puts e.inspect
+        raise AdapterConfigurationError, "Unknown consuming adapter"
+      end
+    end
+  end
+end

data/lib/emque/consuming/adapters/rabbit_mq/manager.rb

@@ -0,0 +1,111 @@
+require "bunny"
+
+module Emque
+  module Consuming
+    module Adapters
+      module RabbitMq
+        class Manager
+          include Emque::Consuming::Actor
+          trap_exit :actor_died
+
+          def actor_died(actor, reason)
+            unless shutdown
+              logger.error "RabbitMQ Manager: actor_died - #{actor.inspect} "+
+                           "died: #{reason}"
+            end
+          end
+
+          def start
+            setup_connection
+            initialize_error_queue
+            initialize_workers
+            logger.info "RabbitMQ Manager: starting #{worker_count} workers..."
+            workers(:flatten => true).each do |worker|
+              worker.async.start
+            end
+          end
+
+          def stop
+            logger.info "RabbitMQ Manager: stopping #{worker_count} workers..."
+
+            super do
+              workers(:flatten => true).each do |worker|
+                logger.info "RabbitMQ Manager: stopping #{worker.topic} worker..."
+                worker.stop
+              end
+            end
+
+            @connection.stop
+          end
+
+          def worker(topic:, command:)
+            if workers.has_key?(topic)
+              case command
+              when :down
+                worker = workers[topic].pop
+                worker.stop if worker
+              when :up
+                workers[topic] << new_worker(topic)
+                workers[topic].last.async.start
+              end
+            end
+          end
+
+          def workers(flatten: false)
+            flatten ? @workers.values.flatten : @workers
+          end
+
+          def retry_errors
+            RetryWorker.new(@connection).retry_errors
+          end
+
+          private
+
+          attr_writer :workers
+
+          def initialize_workers
+            self.workers = {}.tap { |workers|
+              router.topic_mapping.keys.each do |topic|
+                workers[topic] ||= []
+                router.workers(topic).times do
+                  workers[topic] << new_worker(topic)
+                end
+              end
+            }
+          end
+
+          def initialize_error_queue
+            channel = @connection.create_channel
+            error_exchange = channel.fanout(
+              "#{config.app_name}.error",
+              :durable => true,
+              :auto_delete => false
+            )
+            channel.queue(
+              "emque.#{config.app_name}.error",
+              :durable => true,
+              :auto_delete => false,
+              :arguments => {
+                "x-dead-letter-exchange" => "#{config.app_name}.error"
+              }
+            ).bind(error_exchange)
+            channel.close
+          end
+
+          def new_worker(topic)
+            Worker.new_link(@connection, topic)
+          end
+
+          def setup_connection
+            @connection = Bunny.new(config.adapter.options[:url])
+            @connection.start
+          end
+
+          def worker_count
+            workers(:flatten => true).size
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/adapters/rabbit_mq/retry_worker.rb

@@ -0,0 +1,59 @@
+require "bunny"
+
+module Emque
+  module Consuming
+    module Adapters
+      module RabbitMq
+        class RetryWorker
+          include Emque::Consuming::Helpers
+
+          def initialize(connection)
+            self.connection = connection
+            self.channel = connection.create_channel
+          end
+
+          def retry_errors
+            logger.info "RabbitMQ RetryWorker: starting"
+            channel.open if channel.closed?
+            error_queue.message_count.times do
+              delivery_info, properties, payload = error_queue.pop(
+                {:manual_ack => true}
+              )
+              retry_message(delivery_info, properties, payload)
+            end
+            channel.close
+            logger.info "RabbitMQ RetryWorker: done"
+          end
+
+          private
+
+          attr_accessor :connection, :channel
+
+          def error_queue
+            channel.queue(
+              "emque.#{config.app_name}.error",
+              :durable => true,
+              :auto_delete => false,
+              :arguments => {
+                "x-dead-letter-exchange" => "#{config.app_name}.error"
+              }
+            )
+          end
+
+          def retry_message(delivery_info, metadata, payload)
+            begin
+              logger.info "RabbitMQ RetryWorker: processing message #{payload}"
+              message = Emque::Consuming::Message.new(
+                :original => payload
+              )
+              ::Emque::Consuming::Consumer.new.consume(:process, message)
+              channel.ack(delivery_info.delivery_tag)
+            rescue StandardError
+              channel.nack(delivery_info.delivery_tag)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/emque/consuming/adapters/rabbit_mq/worker.rb

@@ -0,0 +1,87 @@
+require "bunny"
+
+module Emque
+  module Consuming
+    module Adapters
+      module RabbitMq
+        class Worker
+          include Emque::Consuming::Actor
+          trap_exit :actor_died
+
+          attr_accessor :topic
+
+          def actor_died(actor, reason)
+            unless shutdown
+              logger.error "#{log_prefix} actor_died - died: #{reason}"
+            end
+          end
+
+          def initialize(connection, topic)
+            self.topic = topic
+            self.name = "#{self.topic} worker"
+            self.shutdown = false
+
+            # @note: channels are not thread safe, so is better to use
+            #        a new channel in each worker.
+            # https://github.com/jhbabon/amqp-celluloid/blob/master/lib/consumer.rb
+            self.channel = connection.create_channel
+
+            if config.adapter.options[:prefetch]
+              channel.prefetch(config.adapter.options[:prefetch])
+            end
+
+            self.queue =
+              channel
+                .queue(
+                  "emque.#{config.app_name}.#{topic}",
+                  :durable => config.adapter.options[:durable],
+                  :auto_delete => config.adapter.options[:auto_delete],
+                  :arguments => {
+                    "x-dead-letter-exchange" => "#{config.app_name}.error"
+                  }
+                )
+                .bind(
+                  channel.fanout(topic, :durable => true, :auto_delete => false)
+                )
+          end
+
+          def start
+            logger.info "#{log_prefix} starting..."
+            queue.subscribe(:manual_ack => true, &method(:process_message))
+            logger.debug "#{log_prefix} started"
+          end
+
+          def stop
+            logger.debug "#{log_prefix} stopping..."
+            super do
+              logger.debug "#{log_prefix} closing channel"
+              channel.close
+            end
+            logger.debug "#{log_prefix} stopped"
+          end
+
+          private
+
+          attr_accessor :name, :channel, :queue
+
+          def process_message(delivery_info, metadata, payload)
+            begin
+              logger.info "#{log_prefix} processing message #{payload}"
+              message = Emque::Consuming::Message.new(
+                :original => payload
+              )
+              ::Emque::Consuming::Consumer.new.consume(:process, message)
+              channel.ack(delivery_info.delivery_tag)
+            rescue StandardError
+              channel.nack(delivery_info.delivery_tag)
+            end
+          end
+
+          def log_prefix
+            "RabbitMQ Worker: #{object_id} #{name}"
+          end
+        end
+      end
+    end
+  end
+end