euston-rabbitmq 1.0.0

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/Gemfile.lock

@@ -0,0 +1,103 @@
+PATH
+  remote: .
+  specs:
+    euston-rabbitmq (1.0.0)
+      activesupport (~> 3.0.0)
+      amqp (~> 0.8.0)
+      bson_ext (~> 1.3.1)
+      euston (~> 1.0.0)
+      euston-eventstore (~> 1.0.0)
+      eventmachine (~> 0.12.10)
+      hash-keys (~> 1.0.0)
+      hollywood (~> 1.0.0)
+      mongo (~> 1.3.1)
+      require_all (~> 1.2.0)
+      robustthread (~> 0.5.2)
+      safely (~> 0.3.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (3.0.10)
+    amq-client (0.8.3)
+      amq-protocol (>= 0.8.0)
+      eventmachine
+    amq-protocol (0.8.1)
+    amqp (0.8.0)
+      amq-client (~> 0.8.3)
+      amq-protocol (~> 0.8.0)
+      eventmachine
+    awesome_print (0.4.0)
+    bson (1.3.1)
+    bson (1.3.1-java)
+    bson_ext (1.3.1)
+    diff-lcs (1.1.3)
+    euston (1.0.0)
+      activesupport (~> 3.0.9)
+      euston-eventstore (~> 1.0.0)
+      require_all (~> 1.2.0)
+    euston-eventstore (1.0.4)
+      activesupport (~> 3.0.9)
+      bson_ext (~> 1.3.1)
+      hash-keys (~> 1.0.0)
+      json (~> 1.5.0)
+      mongo (~> 1.3.1)
+      require_all (~> 1.2.0)
+      uuid (~> 2.3.0)
+    euston-eventstore (1.0.4-java)
+      activesupport (~> 3.0.9)
+      hash-keys (~> 1.0.0)
+      jmongo (~> 1.0.0)
+      json-jruby (~> 1.5.0)
+      require_all (~> 1.2.0)
+      uuid (~> 2.3.0)
+    evented-spec (0.9.0)
+    eventmachine (0.12.10)
+    eventmachine (0.12.10-java)
+    faker (1.0.0)
+      i18n (~> 0.4)
+    fuubar (0.0.6)
+      rspec (~> 2.0)
+      rspec-instafail (~> 0.1.8)
+      ruby-progressbar (~> 0.0.10)
+    hash-keys (1.0.0)
+    hollywood (1.0.0)
+    i18n (0.6.0)
+    jmongo (1.0.3)
+      require_all (~> 1.2)
+    json (1.5.0)
+    json (1.5.0-java)
+    json-jruby (1.5.0-java)
+      json (= 1.5.0)
+    macaddr (1.4.0)
+      systemu (~> 2.2.0)
+    mongo (1.3.1)
+      bson (>= 1.3.1)
+    require_all (1.2.0)
+    robustthread (0.5.2)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-instafail (0.1.8)
+    rspec-mocks (2.6.0)
+    ruby-progressbar (0.0.10)
+    safely (0.3.0)
+    systemu (2.2.0)
+    uuid (2.3.4)
+      macaddr (~> 1.0)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  awesome_print (~> 0.4.0)
+  euston-rabbitmq!
+  evented-spec (~> 0.9.0)
+  faker (~> 1.0.0)
+  fuubar (~> 0.0.0)
+  rspec (~> 2.6.0)

data/Rakefile

@@ -0,0 +1,182 @@
+require 'date'
+require 'rspec/core/rake_task'
+
+#############################################################################
+#
+# Helper functions
+#
+#############################################################################
+
+def name
+  @name ||= Dir['*.gemspec'].first.split('.').first
+end
+
+def version
+  line = File.read("lib/#{name}/version.rb")[/^\s*VERSION\s*=\s*.*/]
+  line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+end
+
+def date
+  Date.today.to_s
+end
+
+def rubyforge_project
+  name
+end
+
+def gemspec_file
+  "#{name}.gemspec"
+end
+
+def gem_file
+  "#{name}-#{version}.gem"
+end
+
+def replace_header(head, header_name, provider = nil)
+  if provider
+    value = send(provider)
+  else
+    value = "'#{send(header_name)}'"
+  end
+
+  provider ||= header_name
+  head.sub!(/(\.#{header_name}\s*= ).*/) { "#{$1}#{value}"}
+end
+
+def platform
+  jruby? ? '-java' : ''
+end
+
+def platform_dependant_gem_file
+  "#{name}-#{version}#{platform}.gem"
+end
+
+def platform_dependent_version
+  "'#{version}#{platform}'"
+end
+
+def jruby?
+  RUBY_PLATFORM.to_s == 'java'
+end
+
+def trim_array_ends array
+  array.shift
+  array.pop
+  array
+end
+
+#############################################################################
+#
+# Custom tasks
+#
+#############################################################################
+
+default_rspec_opts = %w[--colour --format Fuubar]
+
+desc "Run all examples"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = default_rspec_opts
+end
+
+desc "Resets the testing vhost in rabbitmq"
+task :reset_rabbitmq do
+  vhosts = `rabbitmqctl list_vhosts`.split("\n")
+  vhosts = trim_array_ends vhosts
+  vhost = '/euston-rabbitmq-test'
+
+  puts `rabbitmqctl delete_vhost #{vhost}` if vhosts.include? vhost
+  puts `rabbitmqctl add_vhost #{vhost}`
+
+  users = `rabbitmqctl list_users`.split("\n")
+  users = trim_array_ends users
+  users = users.map { |u| u.split("\t").shift }
+
+  user, password = 'euston-rabbitmq-tester', 'password'
+
+  puts `rabbitmqctl delete_user #{user}` if users.include? user
+  puts `rabbitmqctl add_user #{user} #{password}`
+  puts `rabbitmqctl set_permissions -p #{vhost} #{user} ".*" ".*" ".*"`
+  puts `rabbitmqctl set_permissions -p #{vhost} guest ".*" ".*" ".*"` if users.include? 'guest'
+end
+
+#############################################################################
+#
+# Packaging tasks
+#
+#############################################################################
+
+def built_gem
+  @built_gem ||= Dir["#{name}*.gem"].first
+end
+
+desc "Create tag v#{platform_dependent_version} and build and push #{platform_dependant_gem_file} to Rubygems"
+task :release => :build do
+  unless `git branch` =~ /^\* master$/
+    puts "You must be on the master branch to release!"
+    exit!
+  end
+
+  sh "git commit --allow-empty -a -m 'Release #{platform_dependent_version}'"
+  sh "git tag v#{platform_dependent_version}"
+  sh "git push origin master"
+  sh "git push origin v#{platform_dependent_version}"
+
+  command = "gem push pkg/#{platform_dependant_gem_file}"
+
+  if jruby?
+    puts "--------------------------------------------------------------------------------------"
+    puts "can't push to rubygems using jruby at the moment, so switch to mri and run: #{command}"
+    puts "--------------------------------------------------------------------------------------"
+  else
+    sh command
+  end
+end
+
+desc "Build #{platform_dependant_gem_file} into the pkg directory"
+task :build => :gemspec do
+  sh "mkdir -p pkg"
+  sh "gem build #{gemspec_file}"
+  sh "mv #{built_gem} pkg"
+end
+
+desc "Generate #{gemspec_file}"
+task :gemspec => :validate do
+  # read spec file and split out manifest section
+  spec = File.read(gemspec_file)
+  head, manifest, tail = spec.split("  # = MANIFEST =\n")
+
+  # replace name version and date
+  replace_header(head, :name)
+  replace_header(head, :version)
+  replace_header(head, :date)
+  #comment this out if your rubyforge_project has a different name
+  #replace_header(head, :rubyforge_project)
+
+  # determine file list from git ls-files
+  files = `git ls-files`.
+    split("\n").
+    sort.
+    reject { |file| file =~ /^\./ }.
+    reject { |file| file =~ /^(rdoc|pkg)/ }.
+    map { |file| "    #{file}" }.
+    join("\n")
+
+  # piece file back together and write
+  manifest = "  s.files = %w[\n#{files}\n  ]\n"
+  spec = [head, manifest, tail].join("  # = MANIFEST =\n")
+  File.open(gemspec_file, 'w') { |io| io.write(spec) }
+  puts "Updated #{gemspec_file}"
+end
+
+desc "Validate #{gemspec_file}"
+task :validate do
+  libfiles = Dir['lib/*'] - ["lib/#{name}.rb", "lib/#{name}"]
+  unless libfiles.empty?
+    puts "Directory `lib` should only contain a `#{name}.rb` file and `#{name}` dir."
+    exit!
+  end
+  unless Dir['VERSION*'].empty?
+    puts "A `VERSION` file at root level violates Gem best practices."
+    exit!
+  end
+end

data/euston-rabbitmq.gemspec

@@ -0,0 +1,74 @@
+Gem::Specification.new do |s|
+  s.name        = 'euston-rabbitmq'
+  s.version     = '1.0.0'
+  s.platform    = RUBY_PLATFORM.to_s == 'java' ? 'java' : Gem::Platform::RUBY
+  s.authors     = ['Lee Henson', 'Guy Boertje']
+  s.email       = ['lee.m.henson@gmail.com', 'guyboertje@gmail.com']
+  s.summary     = %q{RabbitMq bindings for Euston.}
+  s.description = "JRuby RabbitMq bindings"
+  s.homepage    = 'http://github.com/leemhenson/euston-rabbitmq'
+
+  # = MANIFEST =
+  s.files = %w[
+    Gemfile
+    Gemfile.lock
+    Rakefile
+    euston-rabbitmq.gemspec
+    lib/euston-rabbitmq.rb
+    lib/euston-rabbitmq/buffered_message_dispatcher.rb
+    lib/euston-rabbitmq/command_handler_bindings.rb
+    lib/euston-rabbitmq/command_handlers/retry_failed_message.rb
+    lib/euston-rabbitmq/errors.rb
+    lib/euston-rabbitmq/event_handler_binding.rb
+    lib/euston-rabbitmq/event_handler_bindings.rb
+    lib/euston-rabbitmq/event_handlers/message_failure.rb
+    lib/euston-rabbitmq/event_store_dispatcher.rb
+    lib/euston-rabbitmq/exchanges.rb
+    lib/euston-rabbitmq/handler_bindings.rb
+    lib/euston-rabbitmq/message_buffer.rb
+    lib/euston-rabbitmq/message_logger.rb
+    lib/euston-rabbitmq/queue.rb
+    lib/euston-rabbitmq/queues.rb
+    lib/euston-rabbitmq/read_model/failed_message.rb
+    lib/euston-rabbitmq/read_model/message_buffer.rb
+    lib/euston-rabbitmq/read_model/message_log.rb
+    lib/euston-rabbitmq/version.rb
+    spec/command_buffer_spec.rb
+    spec/event_buffer_spec.rb
+    spec/exchange_declaration_spec.rb
+    spec/message_failure_spec.rb
+    spec/mt_safe_queue_subscription_spec.rb
+    spec/safe_queue_subscription_spec.rb
+    spec/spec_helper.rb
+    spec/support/factories.rb
+  ]
+  # = MANIFEST =
+
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+
+  s.add_dependency 'activesupport',               '~> 3.0.0'
+  s.add_dependency 'euston',                      '~> 1.0.0'
+  s.add_dependency 'euston-eventstore',           '~> 1.0.0'
+  s.add_dependency 'hash-keys',                   '~> 1.0.0'
+  s.add_dependency 'hollywood',                   '~> 1.0.0'
+  s.add_dependency 'require_all',                 '~> 1.2.0'
+  s.add_dependency 'robustthread',                '~> 0.5.2'
+  s.add_dependency 'safely',                      '~> 0.3.0'
+
+  if RUBY_PLATFORM.to_s == 'java'
+    s.add_dependency 'jmongo',                    '~> 1.0.0'
+    s.add_dependency 'jessica',                   '~> 1.0.0'
+  else
+    s.add_dependency 'amqp',                      '~> 0.8.0'
+    s.add_dependency 'bson_ext',                  '~> 1.3.1'
+    s.add_dependency 'eventmachine',              '~> 0.12.10'
+    s.add_dependency 'mongo',                     '~> 1.3.1'
+    s.add_development_dependency 'evented-spec',  '~> 0.9.0'
+  end
+
+  s.add_development_dependency 'awesome_print',   '~> 0.4.0'
+  s.add_development_dependency 'faker',           '~> 1.0.0'
+  s.add_development_dependency 'fuubar',          '~> 0.0.0'
+  s.add_development_dependency 'rspec',           '~> 2.6.0'
+end

data/lib/euston-rabbitmq/buffered_message_dispatcher.rb

@@ -0,0 +1,48 @@
+module Euston
+  module RabbitMq
+    class BufferedMessageDispatcher
+      class << self
+        def command_dispatcher channel
+          @command_dispatcher ||= BufferedMessageDispatcher.new(Euston::RabbitMq::MessageBuffer.commands_buffer channel)
+        end
+
+        def event_dispatcher channel
+          @event_dispatcher ||= BufferedMessageDispatcher.new(Euston::RabbitMq::MessageBuffer.events_buffer channel)
+        end
+      end
+
+      include Euston::RabbitMq::Exchanges
+
+      def initialize buffer
+        @buffer = buffer
+        @buffer.when(:no_messages_were_due) { @sleep = true }
+      end
+
+      attr_writer :wait_time
+
+      def wait_time
+        @wait_time ||= 1
+      end
+
+      def stop
+        @stop = true
+      end
+
+      def start
+        @stop = false
+
+        dispatch_loop = Proc.new do
+          @sleep = false
+
+          begin
+            @buffer.dispatch_due_messages
+          end while !@sleep
+
+          EM.add_timer(wait_time) { dispatch_loop.call } unless @stop
+        end
+
+        EM.add_timer(wait_time) { dispatch_loop.call }
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/command_handler_bindings.rb

@@ -0,0 +1,30 @@
+module Euston
+  module RabbitMq
+    class CommandHandlerBindings < HandlerBindings
+      def finalize_bindings
+        super
+        attach_queue_hook_listeners queue
+      end
+
+      protected
+
+      def bind_handler handler_type
+        queue.bind @exchanges[:commands], :routing_key => "commands.#{handler_type.to_s.underscore}"
+      end
+
+      def call_handler message
+        command_headers = CommandHeaders.from_hash(message[:headers]).freeze
+        command_body = message[:body].freeze
+        handler_type = command_headers.type.to_s.camelize.to_sym
+        find_namespaces_containing_handler_type(handler_type).each do |namespace|
+          handler = namespace.const_get handler_type
+          handler.new.send "__version__#{command_headers.version}", command_headers, command_body
+        end
+      end
+
+      def queue
+        @queue ||= get_queue @channel, :command_handlers
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/command_handlers/retry_failed_message.rb

@@ -0,0 +1,29 @@
+module Euston
+  module RabbitMq
+    module CommandHandlers
+      class RetryFailedMessage
+        include Euston::CommandHandler
+
+        version 1 do |headers, command|
+          model = Euston::RabbitMq::ReadModel::FailedMessage.new
+          message = model.get_by_id command[:id]
+
+          unless message.nil?
+            routing_key = message['routing_key']
+            exchange = routing_key.split('.').first
+
+            headers = message['headers'].merge(:id => command[:id],
+                                               :type => message['type'],
+                                               :version => message['version'] )
+
+            buffer = Euston::RabbitMq::MessageBuffer.send("#{exchange}_buffer")
+            buffer.push :headers => headers,
+                        :body => message['body']
+
+            model.remove_by_id command[:id]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/errors.rb

@@ -0,0 +1,5 @@
+module Euston
+  module RabbitMq
+    class MessageDecodeFailedError < StandardError; end
+  end
+end

data/lib/euston-rabbitmq/event_handler_binding.rb

@@ -0,0 +1,114 @@
+module Euston
+  module RabbitMq
+    class EventHandlerBinding
+      include Euston::RabbitMq::Queues
+      include Euston::RabbitMq::Exchanges
+
+      PREFIX = '__event_handler__'
+
+      def initialize(channel, namespace, handler_type)
+        @channel, @namespace, @handler_type = channel, namespace, handler_type
+
+        name = @handler_type.to_s.underscore
+        @queue = get_queue(@channel, name)
+        instance_variable_set "@#{name}".to_sym, @queue
+        @handler_class = @namespace.const_get @handler_type
+        @exchange = get_exchange(@channel, :events)
+      end
+
+      def bind
+        bindable_methods.each do |method|
+          @queue.bind @exchange, :routing_key => "events.#{method}"
+        end
+        self
+      end
+
+      def listen
+        attach_queue_hook_listeners @queue
+      end
+
+      private
+
+      def attach_queue_hook_listeners queue
+        queue.when(:message_decode_failed => method(:log_decode_failure),
+                   :message_failed        => method(:handle_failure),
+                   :message_received      => method(:call_handler))
+
+        queue.safe_subscribe
+      end
+
+      def call_handler(message)
+        event_headers = EventHeaders.from_hash message[:headers]
+        event_body = message[:body]
+
+        @handler_class.new.send "#{PREFIX}#{event_headers.type}__#{event_headers.version}", event_headers.freeze, event_body.freeze
+      end
+
+      def bindable_methods
+        methods = @handler_class.public_instance_methods(false)
+        methods = methods.select { |method| method.to_s.start_with? PREFIX }
+        methods = methods.map { |method| method.to_s[PREFIX.length, method.to_s.length - PREFIX.length] }
+        methods = methods.map { |method| method.split('__').first }
+        methods.uniq
+      end
+
+      def create_message_failed_message error, failed_message, amqp_header
+        {
+          :headers =>
+          {
+            :id => Euston.uuid.generate.to_s,
+            :type => :message_failed,
+            :version => 1,
+            :timestamp => Time.now.to_f
+          },
+
+          :body =>
+          {
+            :message => failed_message,
+            :routing_key => amqp_header.method.routing_key,
+            :error => error.message,
+            :backtrace => error.backtrace
+          }
+        }
+      end
+
+      def handle_failure message, error, amqp_header
+        failures = message[:failures] || 0
+        failures = failures + 1
+        message[:failures] = failures
+
+        begin
+          if failures == 3
+            message.delete :failures
+            message = create_message_failed_message error, message, amqp_header
+
+            options = { :key => 'events.message_failed' }
+            exchange_ref = :events
+          else
+            options = { :key => amqp_header.method.routing_key }
+            exchange_ref = amqp_header.method.routing_key.split('.').first.to_sym
+          end
+
+          options = default_publish_options.merge options
+          exchange = get_exchange(@channel, exchange_ref)
+          message = ::ActiveSupport::JSON.encode message
+
+          exchange.publish message, options
+
+          amqp_header.ack
+        rescue StandardError => e
+          amqp_header.reject :requeue => true
+          raise e
+        end
+      end
+
+      def log_decode_failure message, error
+        text = "A handler queue subscription failed. [Error] #{error.message} [Payload] #{message}"
+        err = Euston::RabbitMq::MessageDecodeFailedError.new text
+        err.set_backtrace error.backtrace
+
+        Safely.report! err
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/event_handler_bindings.rb

@@ -0,0 +1,30 @@
+module Euston
+  module RabbitMq
+    class EventHandlerBindings < HandlerBindings
+      PREFIX = '__event_handler__'
+
+      protected
+
+      def bind_handler handler_type
+        name = handler_type.to_s.underscore
+        queue = get_queue @channel, name
+        instance_variable_set "@#{name}".to_sym, queue
+
+        namespace = find_namespaces_containing_handler_type(handler_type).first
+        handler_class = namespace.const_get handler_type
+        methods = handler_class.public_instance_methods(false)
+        methods = methods.select { |method| method.to_s.start_with? PREFIX }
+        methods = methods.map { |method| method.to_s[PREFIX.length, method.to_s.length - PREFIX.length] }
+        methods = methods.map { |method| method.split('__').first }
+        methods.uniq.each { |method| queue.bind @exchanges[:events], :routing_key => "events.#{method}" }
+
+        attach_queue_hook_listeners queue do |message|
+          event_headers = EventHeaders.from_hash message[:headers]
+          event_body = message[:body]
+
+          handler_class.new.send "#{PREFIX}#{event_headers.type}__#{event_headers.version}", event_headers.freeze, event_body.freeze
+        end
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/event_handlers/message_failure.rb

@@ -0,0 +1,27 @@
+module Euston
+  module RabbitMq
+    module EventHandlers
+      class MessageFailure
+        include Euston::EventHandler
+
+        consumes :message_failed, 1 do |headers, event|
+          headers = event[:message][:headers].dup
+          failure = { :message_id => headers.delete(:id),
+                      :type => headers.delete(:type),
+                      :version => headers.delete(:version),
+                      :message_timestamp => headers.delete(:timestamp),
+                      :routing_key => event[:routing_key],
+                      :body => event[:message][:body],
+                      :headers => headers,
+                      :error => event[:error],
+                      :backtrace => event[:backtrace],
+                      :failure_timestamp => Time.now.to_f }
+
+          model = Euston::RabbitMq::ReadModel::FailedMessage
+          model = model.new
+          model.log_failure failure
+        end
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/event_store_dispatcher.rb

@@ -0,0 +1,45 @@
+module Euston
+  module RabbitMq
+    class EventStoreDispatcher
+      def initialize channel
+        @channel = channel
+        @event_buffer = Euston::RabbitMq::MessageBuffer.events_buffer channel
+        @event_store = Euston::RabbitMq.event_store
+      end
+
+      attr_writer :wait_time
+
+      def wait_time
+        @wait_time ||= 1
+      end
+
+      def stop
+        @stop = true
+      end
+
+      def start
+        @stop = false
+
+        dispatch_loop = Proc.new do
+          begin
+            begin
+              commits = @event_store.get_undispatched_commits
+
+              commits.each do |commit|
+                commit.events.each { |event| @event_buffer.push event.to_hash }
+
+                @event_store.mark_commit_as_dispatched commit
+              end
+            end while !commits.empty?
+          rescue StandardError => e
+            DaemonKit::logger.error "#{e}\n#{e.backtrace}"
+          end
+
+          EM.add_timer(wait_time) { dispatch_loop.call } unless @stop
+        end
+
+        EM.add_timer(wait_time) { dispatch_loop.call }
+      end
+    end
+  end
+end