mantle 2.0.0

data/lib/mantle/configuration.rb

@@ -0,0 +1,27 @@
+require 'logger'
+
+module Mantle
+  class Configuration
+
+    attr_accessor :message_bus_channels,
+      :message_bus_redis,
+      :message_handler,
+      :logger,
+      :redis_namespace
+
+    def initialize
+      @message_bus_channels = []
+      @message_handler = Mantle::MessageHandler
+      @logger = default_logger
+      @redis_namespace = nil
+    end
+
+    private
+
+    def default_logger
+      logger = Logger.new(STDOUT)
+      logger.level = Logger::INFO
+      logger
+    end
+  end
+end

data/lib/mantle/error.rb

@@ -0,0 +1,6 @@
+module Mantle
+  module Error
+    MissingRedisConnection = Class.new(StandardError)
+    MissingImplementation = Class.new(StandardError)
+  end
+end

data/lib/mantle/local_redis.rb

@@ -0,0 +1,42 @@
+module Mantle
+  class LocalRedis
+    SUCCESSFUL_MESSAGE_KEY = "last_successful_message_received"
+    CATCH_UP_CLEANUP_KEY = "mantle:catch_up:cleanup"
+
+    def self.set_message_successfully_received(time = Time.now.utc.to_f.to_s)
+      Sidekiq.redis { |conn| conn.set(SUCCESSFUL_MESSAGE_KEY, time) }
+      Mantle.logger.debug("Set last successful message received time: #{time}")
+      process_redis_response(time)
+    end
+
+    def self.last_message_successfully_received_at
+      result = Sidekiq.redis { |conn| conn.get(SUCCESSFUL_MESSAGE_KEY) }
+      process_redis_response(result)
+    end
+
+    def self.set_catch_up_cleanup(time = Time.now.utc.to_f.to_s)
+      Sidekiq.redis { |conn| conn.set(CATCH_UP_CLEANUP_KEY, time) }
+      Mantle.logger.debug("Set last catch up cleanup time: #{time}")
+      process_redis_response(time)
+    end
+
+    def self.last_catch_up_cleanup_at
+      result = Sidekiq.redis { |conn| conn.get(CATCH_UP_CLEANUP_KEY) }
+      process_redis_response(result)
+    end
+
+    private
+
+    def self.process_redis_response(result)
+      if result.nil? || result == ""
+        nil
+      else
+        result.to_f
+      end
+    end
+
+    def self.get(key)
+      Sidekiq.redis { |conn| conn.get(key) }
+    end
+  end
+end

data/lib/mantle/message.rb

@@ -0,0 +1,21 @@
+module Mantle
+  class Message
+    attr_reader :channel
+    attr_writer :message_bus, :catch_up
+
+    def initialize(channel)
+      @channel = channel
+      @message_bus = Mantle::MessageBus.new
+      @catch_up = Mantle::CatchUp.new
+    end
+
+    def publish(message)
+      message_bus.publish(channel, message)
+      catch_up.add_message(channel, message)
+    end
+
+    private
+
+    attr_reader :message_bus, :catch_up
+  end
+end

data/lib/mantle/message_bus.rb

@@ -0,0 +1,44 @@
+module Mantle
+  class MessageBus
+    attr_writer :redis
+
+    def initialize
+      @redis = Mantle.configuration.message_bus_redis
+      @channels = Mantle.configuration.message_bus_channels
+    end
+
+    def publish(channel, message)
+      json = JSON.generate(message)
+      redis.publish(channel, json)
+      Mantle.logger.debug("Sent message to message bus channel: #{channel}")
+    end
+
+    def listen
+      Mantle.logger.info("Connecting to message bus redis: #{redis.inspect} ")
+
+      catch_up
+      subscribe_to_channels
+    end
+
+    def catch_up
+      Mantle::CatchUp.new.catch_up
+    end
+
+    def subscribe_to_channels
+      raise Mantle::Error::MissingRedisConnection unless redis
+
+      Mantle.logger.info("Subscribing to message bus for #{channels} ")
+
+      redis.subscribe(channels) do |on|
+        on.message do |channel, json_message|
+          message = JSON.parse(json_message)
+          Mantle::MessageRouter.new(channel, message).route
+        end
+      end
+    end
+
+    private
+
+    attr_reader :redis, :channels
+  end
+end

data/lib/mantle/message_handler.rb

@@ -0,0 +1,7 @@
+module Mantle
+  class MessageHandler
+    def self.receive(channel, message)
+      raise Mantle::Error::MissingImplementation.new("Implement self.receive(channel, object) and assign class to the message handler")
+    end
+  end
+end

data/lib/mantle/message_router.rb

@@ -0,0 +1,16 @@
+module Mantle
+  class MessageRouter
+    def initialize(channel, message)
+      @channel, @message = channel, message
+    end
+
+    def route
+      return unless @message
+
+      Mantle.logger.debug("Routing message for #{@channel}")
+      Mantle.logger.debug("Message: #{@message}")
+
+      Mantle::Workers::ProcessWorker.perform_async(@channel, @message)
+    end
+  end
+end

data/lib/mantle/railtie.rb

@@ -0,0 +1,6 @@
+require 'rails'
+
+module Mantle
+  class Railtie < Rails::Railtie
+  end
+end

data/lib/mantle/version.rb

@@ -0,0 +1,3 @@
+module Mantle
+  VERSION = '2.0.0'
+end

data/lib/mantle/workers/catch_up_cleanup_worker.rb

@@ -0,0 +1,15 @@
+module Mantle
+  module Workers
+    class CatchUpCleanupWorker
+      include Sidekiq::Worker
+
+      sidekiq_options queue: :mantle
+
+      def perform
+        Mantle::CatchUp.new.clear_expired
+        Mantle::LocalRedis.set_catch_up_cleanup
+      end
+    end
+  end
+end
+

data/lib/mantle/workers/process_worker.rb

@@ -0,0 +1,15 @@
+module Mantle
+  module Workers
+    class ProcessWorker
+      include Sidekiq::Worker
+
+      sidekiq_options queue: :mantle
+
+      def perform(channel, message)
+        Mantle.receive_message(channel, message)
+        Mantle::LocalRedis.set_message_successfully_received
+        Mantle::CatchUp.new.enqueue_clear_if_ready
+      end
+    end
+  end
+end

data/mantle.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mantle/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "mantle"
+  gem.version       = Mantle::VERSION
+  gem.authors       = ["Grant Ammons", "Brandon Hilkert"]
+  gem.email         = ["gammons@gmail.com", "brandonhilkert@gmail.com"]
+  gem.description   = %q{Ruby application message bus subscriptions with Sidekiq and Redis Pubsub.}
+  gem.summary       = %q{Ruby application message bus subscriptions with Sidekiq and Redis Pubsub.}
+  gem.homepage      = "https://github.com/PipelineDeals/mantle"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency('redis')
+  gem.add_dependency('sidekiq')
+
+  gem.add_development_dependency('rspec')
+  gem.add_development_dependency('pry')
+end

data/spec/lib/mantle/catch_up_spec.rb

@@ -0,0 +1,174 @@
+require 'spec_helper'
+
+describe Mantle::CatchUp do
+  let(:handler) { Mantle::CatchUp.new }
+  let(:redis) { handler.redis }
+
+  before :each do
+    Mantle.logger.level = Logger::WARN
+    Mantle.configuration.message_bus_redis.flushdb
+  end
+
+  after :each do
+    Mantle.configuration.message_bus_redis.flushdb
+  end
+
+  describe "#add_message" do
+    it "adds message to redis that expires in 6 hours" do
+      channel = "person:update"
+      message = { id: 1 }
+      json_message = JSON.generate(message)
+
+      catch_up = Mantle::CatchUp.new
+      catch_up.add_message(channel, message)
+
+      json_payload = redis.zrange(catch_up.key, 0, -1).first
+      channel, message = catch_up.deserialize_payload(json_payload)
+
+      expect(channel).to eq(channel)
+      expect(message).to eq(message)
+    end
+
+    it "returns time for message" do
+      catch_up = Mantle::CatchUp.new
+      time = catch_up.add_message("person:update", { id: 1 }, 1234.56)
+      expect(time).to eq(1234.56)
+    end
+  end
+
+  describe "#enqueue_clear_if_ready" do
+    it "enqueues clear job if it was done more than 5 min. ago" do
+      time = Time.now.utc.to_f - ((Mantle::CatchUp::CLEANUP_EVERY_MINUTES + 1) * 60.0)
+      Mantle::LocalRedis.set_catch_up_cleanup(time)
+
+      expect {
+        Mantle::CatchUp.new.enqueue_clear_if_ready
+      }.to change(Mantle::Workers::CatchUpCleanupWorker.jobs, :size).by(1)
+    end
+
+    it "enqueues clear job if no last cleanup time has been recorded" do
+      Mantle::LocalRedis.set_catch_up_cleanup(nil)
+
+      expect {
+        Mantle::CatchUp.new.enqueue_clear_if_ready
+      }.to change(Mantle::Workers::CatchUpCleanupWorker.jobs, :size).by(1)
+    end
+
+    it "doesn't enqueue a clear job if enough time hasn't passed" do
+      time = Time.now.utc.to_f - ((Mantle::CatchUp::CLEANUP_EVERY_MINUTES - 1) * 60.0)
+      Mantle::LocalRedis.set_catch_up_cleanup(time)
+
+      expect {
+        Mantle::CatchUp.new.enqueue_clear_if_ready
+      }.to_not change(Mantle::Workers::CatchUpCleanupWorker.jobs, :size)
+    end
+  end
+
+  describe "#clear_expired" do
+    it "clears expired entries from the catch up list" do
+      cu = Mantle::CatchUp.new
+      cu.add_message("person:update", { id: 1 }, cu.hours_ago_in_seconds(8))
+      cu.add_message("deal:update", { id: 2 }, cu.hours_ago_in_seconds(7))
+      cu.add_message("company:update", { id: 3 }, cu.hours_ago_in_seconds(5))
+
+      cu.clear_expired
+
+      expect(redis.zcount(cu.key, 0, 'inf')).to eq(1)
+
+      json_payload = redis.zrange(cu.key, 0, -1).first
+      channel, message = cu.deserialize_payload(json_payload)
+
+      expect(channel).to eq("company:update")
+    end
+  end
+
+  describe "catch_up" do
+    it "raises when redis connection is missing" do
+      cu = Mantle::CatchUp.new
+      cu.redis = nil
+
+      expect {
+        cu.catch_up
+      }.to raise_error(Mantle::Error::MissingRedisConnection)
+    end
+
+    it "skips if no successfully processed time has been recorded" do
+      cu = Mantle::CatchUp.new
+
+      cu.add_message("person:update", { id: 1 })
+      cu.add_message("deal:update", { id: 2 })
+      cu.add_message("company:update", { id: 3 })
+      time = cu.add_message("user:update", { id: 3 })
+
+      expect(cu).to_not receive(:route_messages)
+
+      cu.catch_up
+    end
+
+    it "doesn't process anything when system is up to date no last successfully processed message time has been record" do
+      cu = Mantle::CatchUp.new
+
+      cu.add_message("person:update", { id: 1 })
+      cu.add_message("deal:update", { id: 2 })
+      cu.add_message("company:update", { id: 3 })
+      time = cu.add_message("user:update", { id: 3 })
+
+      Mantle::LocalRedis.set_message_successfully_received
+
+      expect(cu).to_not receive(:route_messages)
+
+      cu.catch_up
+    end
+
+    it "handles all messages that need catch up" do
+      cu = Mantle::CatchUp.new
+
+      cu.add_message("person:update", { id: 1 })
+      cu.add_message("deal:update", { id: 2 })
+      cu.add_message("company:update", { id: 3 })
+
+      Mantle::LocalRedis.set_message_successfully_received
+
+      time = cu.add_message("user:update", { id: 3 })
+
+      expect(cu).to receive(:route_messages).with(
+        [["{\"channel\":\"user:update\",\"message\":{\"id\":3}}", time]]
+      )
+
+      cu.catch_up
+    end
+  end
+
+  describe "#last_success_time" do
+    it "returns time of last successfully process message"do
+      expect(Mantle::LocalRedis).to receive(:last_message_successfully_received_at)
+      Mantle::CatchUp.new.last_success_time
+    end
+  end
+
+  describe "#route_messages" do
+    it "process messages if listening to that channel" do
+      p =[["{\"channel\":\"user:update\",\"message\":{\"id\":3}}", 1423336645.314663]]
+      cu = Mantle::CatchUp.new
+      cu.message_bus_channels = ["user:update"]
+
+      expect(Mantle::MessageRouter).to receive(:new).with(
+        "user:update", { "id" => 3 }
+      ).and_return(double("router", route: true))
+
+      cu.route_messages(p)
+    end
+
+    it "skips messages on channels not subscribed" do
+      p =[["{\"channel\":\"user:update\",\"message\":{\"id\":3}}", 1423336645.314663]]
+      cu = Mantle::CatchUp.new
+      cu.message_bus_channels = ["user:create"]
+
+      expect(Mantle::MessageRouter).to_not receive(:new)
+
+      cu.route_messages(p)
+    end
+  end
+
+end
+

data/spec/lib/mantle/configuration_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+describe Mantle::Configuration do
+  it 'can set/get message_bus_channels' do
+    config = Mantle::Configuration.new
+    config.message_bus_channels = ["update"]
+    expect(config.message_bus_channels).to eq(Array("update"))
+  end
+
+  it 'sets default message_bus_channels' do
+    config = Mantle::Configuration.new
+    expect(config.message_bus_channels).to eq([])
+  end
+
+  it 'can set/get message_bus_redis' do
+    redis = double("redis")
+    config = Mantle::Configuration.new
+    config.message_bus_channels = redis
+    expect(config.message_bus_channels).to eq(redis)
+  end
+
+  it 'can set/get message_handler' do
+    FakeHandler = Class.new
+
+    config = Mantle::Configuration.new
+    config.message_handler = FakeHandler
+    expect(config.message_handler).to eq(FakeHandler)
+  end
+
+  it 'configures default message handler' do
+    config = Mantle::Configuration.new
+    expect(config.message_handler).to eq(Mantle::MessageHandler)
+  end
+
+  it 'can set/get logger' do
+    logger = Logger.new(STDOUT)
+    config = Mantle::Configuration.new
+    config.logger = logger
+    expect(config.logger).to eq(logger)
+  end
+
+  it 'configures default logger' do
+    config = Mantle::Configuration.new
+    expect(config.logger.level).to eq(1)
+  end
+
+  it 'can set/get namespace for local redis listen' do
+    config = Mantle::Configuration.new
+    config.redis_namespace = "fake"
+    expect(config.redis_namespace).to eq("fake")
+  end
+
+  it 'configures default redis namespace' do
+    config = Mantle::Configuration.new
+    expect(config.redis_namespace).to eq(nil)
+  end
+end
+
+