mantle 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9faf822274115317d82c5b0e254b7551df79b3f5
+  data.tar.gz: 2f9d06db2359478ca001df1b124eae4a46f84511
+SHA512:
+  metadata.gz: 8aa2b4e2059be5955e1f3cf033e0a775db5ffc29fb54947887a709024f9bc8e774ffc00b23d8fc9071685cbaff906b80459ce4908244efd873b981f36703a63f
+  data.tar.gz: 946357abc8c54760e04b0b94f2f66c0238279181fa1b7cda362ae294a20bbf799104079662ef282e9ed6e5ba81aad2a73d600ebe0feefac7aaf13200e7106ad0

data/.gitignore

@@ -0,0 +1,18 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile ~/.gitignore_global
+
+# Ignore bundler config
+/.bundle
+
+# Ignore the default SQLite database.
+/db/*.sqlite3
+
+# Ignore all logfiles and tempfiles.
+/log/*.log
+/tmp
+
+.DS_Store
+.idea

data/.rspec

@@ -0,0 +1 @@
+--colour

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,55 @@
+PATH
+  remote: .
+  specs:
+    mantle (2.0.0)
+      redis
+      sidekiq
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    coderay (1.1.0)
+    connection_pool (2.1.2)
+    diff-lcs (1.2.5)
+    hitimes (1.2.2)
+    json (1.8.2)
+    method_source (0.8.2)
+    pry (0.10.0)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    redis (3.2.1)
+    redis-namespace (1.5.1)
+      redis (~> 3.0, >= 3.0.4)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.2)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    sidekiq (3.3.2)
+      celluloid (>= 0.16.0)
+      connection_pool (>= 2.1.1)
+      json
+      redis (>= 3.0.6)
+      redis-namespace (>= 1.3.1)
+    slop (3.5.0)
+    timers (4.0.1)
+      hitimes
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  mantle!
+  pry
+  rspec

data/README.md

@@ -0,0 +1,107 @@
+# Mantle
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mantle'
+
+or install manually by:
+
+    $ gem install mantle
+
+
+## Usage (in Rails App)
+
+Setup a Rails initializer(`config/initializers/mantle.rb`):
+
+
+```Ruby
+require_relative '../../app/models/mantle_message_handler'
+
+Mantle.configure do |config|
+  config.message_bus_channels = %w[account:update orders]
+  config.message_bus_redis = Redis.new(host: ENV["MESSAGE_BUS_REDIS_URL"] || 'localhost')
+  config.message_handler = MantleMessageHandler
+end
+```
+
+The config takes a number of options, many of which have defaults:
+
+```Ruby
+Mantle.configure do |config|
+  config.message_bus_channels = ['deal:update', 'create:person'] # default: []
+  config.message_bus_redis = Redis.new(host: 'localhost') # default: localhost
+  config.message_handler = MyMessageHandler # requires implementation
+  config.logger = Rails.logger # default: Logger.new(STDOUT)
+  config.redis_namespace = "my-namespace" # default: no namespace
+end
+```
+
+To make the installation of mantle easier, the following command will create
+these files in a Rails application:
+
+```
+$ rails g mantle:install
+```
+
+If an application only pushes messages on to the queue and doesn't listen, the
+following configuration is all that's needed:
+
+```Ruby
+Mantle.configure do |config|
+  config.message_bus_redis = Redis.new(host: 'localhost') # default: localhost
+  config.logger = Rails.logger # default: Logger.new(STDOUT)
+end
+```
+
+
+Publish messages to consumers:
+
+```Ruby
+Mantle::Message.new("person:create").publish({ id: message['id'], data: message['data'] })
+```
+
+The first and only argument to `Mantle::Message.new` is the channel you want to publish the
+message on. The `#publish` method takes the message payload (in any format you like)
+and pushes the message on to the message bus pub/sub and also adds it to the
+catch up queue so offline applications can process the message when they become available.
+
+Define message handler class with `.receive` method. For example `app/models/my_message_handler.rb`
+
+```Ruby
+class MyMessageHandler
+  def self.receive(channel, message)
+    puts channel # => 'order'
+    puts message # => { 'id' => 5, 'name' => 'Brandon' }
+  end
+end
+```
+
+To run the listener:
+
+```
+$ bin/mantle
+```
+
+or with configuration:
+
+```
+$ bin/mantle -c ./config/initializers/other_file.rb
+```
+
+To run the processor:
+
+```
+$ bin/sidekiq -q mantle
+```
+
+If the Sidekiq worker should also listen on another queue, add that to the
+command with:
+
+```
+$ bin/sidekiq -q mantle -q default
+```
+
+It will NOT add the `default` queue to processing if there are other queues
+enumerated using the `-q` option.

data/Rakefile

@@ -0,0 +1,5 @@
+require "bundler/gem_tasks"
+
+task :console do
+  exec "irb -r mantle -I ./lib"
+end

data/bin/mantle

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'celluloid'
+require 'sidekiq/cli'
+require 'sidekiq/processor'
+
+require_relative '../lib/mantle/cli'
+
+cli = Mantle::CLI.new
+cli.setup
+
+trap('SIGINT') { exit! }
+
+cli.listen

data/circle.yml

@@ -0,0 +1,3 @@
+machine:
+  ruby:
+    version: '2.2'

data/config.ru

@@ -0,0 +1,7 @@
+require_relative './lib/mantle'
+require 'sidekiq'
+
+Mantle.boot_system!
+
+require 'sidekiq/web'
+run Sidekiq::Web

data/lib/generators/mantle/install/install_generator.rb

@@ -0,0 +1,20 @@
+module Mantle
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("../templates", __FILE__)
+
+      desc <<desc
+description:
+    copy mantle config to a Rails initializer and create default handler
+desc
+
+      def create_configuration
+        template "mantle.rb", "config/initializers/mantle.rb"
+      end
+
+      def create_handler
+        template "mantle_message_handler.rb", "app/models/mantle_message_handler.rb"
+      end
+    end
+  end
+end

data/lib/generators/mantle/install/templates/mantle.rb

@@ -0,0 +1,9 @@
+# require 'sidekiq-pro'
+require_relative '../../app/models/mantle_message_handler'
+
+Mantle.configure do |config|
+  config.message_bus_channels = %w[]
+  config.message_bus_redis = Redis.new(host: ENV["MESSAGE_BUS_REDIS_URL"] || 'localhost')
+  config.message_handler = MantleMessageHandler
+end
+

data/lib/generators/mantle/install/templates/mantle_message_handler.rb

@@ -0,0 +1,7 @@
+class MantleMessageHandler
+  def self.receive(channel, message)
+    puts channel # => 'order'
+    puts message # => { 'id' => 5, 'name' => 'Brandon' }
+  end
+end
+

data/lib/mantle.rb

@@ -0,0 +1,45 @@
+require 'rubygems'
+require 'redis'
+require 'sidekiq'
+require 'json'
+
+begin
+  require 'pry'
+rescue LoadError
+end
+
+require_relative 'mantle/catch_up'
+require_relative 'mantle/configuration'
+require_relative 'mantle/error'
+require_relative 'mantle/local_redis'
+require_relative 'mantle/message'
+require_relative 'mantle/message_bus'
+require_relative 'mantle/message_handler'
+require_relative 'mantle/message_router'
+require_relative 'mantle/workers/catch_up_cleanup_worker'
+require_relative 'mantle/workers/process_worker'
+require_relative 'mantle/version'
+
+require_relative 'mantle/railtie' if defined?(Rails)
+
+module Mantle
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration) if block_given?
+  end
+
+  def self.receive_message(channel, message)
+    Mantle.logger.debug("Message received on channel: #{channel}")
+    Mantle.logger.debug("Mantle message: #{message}")
+
+    self.configuration.message_handler.receive(channel, message)
+  end
+
+  def self.logger
+    configuration.logger
+  end
+end

data/lib/mantle/catch_up.rb

@@ -0,0 +1,84 @@
+module Mantle
+  class CatchUp
+    KEY = "mantle:catch_up"
+    HOURS_TO_KEEP = 6
+    CLEANUP_EVERY_MINUTES = 5
+
+    attr_accessor :redis, :message_bus_channels
+    attr_reader :key
+
+    def initialize
+      @redis = Mantle.configuration.message_bus_redis
+      @message_bus_channels = Mantle.configuration.message_bus_channels
+      @key = KEY
+    end
+
+    def add_message(channel, message, now = Time.now.utc.to_f)
+      json = serialize_payload(channel, message)
+      redis.zadd(key, now, json)
+      Mantle.logger.debug("Added message to catch up list for channel: #{channel}")
+      now
+    end
+
+    def enqueue_clear_if_ready
+      now = Time.now.utc.to_f
+      five_minutes_ago = now - (CLEANUP_EVERY_MINUTES * 60.0)
+      last_cleanup = Mantle::LocalRedis.last_catch_up_cleanup_at
+
+      if last_cleanup.nil? || last_cleanup < five_minutes_ago
+        Mantle::Workers::CatchUpCleanupWorker.perform_async
+      end
+    end
+
+    def clear_expired
+      max_time_to_clear = hours_ago_in_seconds(HOURS_TO_KEEP)
+      redis.zremrangebyscore(key, 0 , max_time_to_clear)
+    end
+
+    def catch_up
+      raise Mantle::Error::MissingRedisConnection unless redis
+
+      if last_success_time.nil?
+        Mantle.logger.info("Skipping catch up because of missing last processed time...")
+        return
+      end
+
+      Mantle.logger.info("Catching up from time: #{last_success_time}")
+
+      payloads_with_time = redis.zrangebyscore(key, last_success_time, 'inf', with_scores: true)
+      route_messages(payloads_with_time) if payloads_with_time.any?
+    end
+
+    def last_success_time
+      LocalRedis.last_message_successfully_received_at
+    end
+
+    def route_messages(payloads_with_time)
+      payloads_with_time.each do |payload_with_time|
+        payload, time = payload_with_time
+        channel, message = deserialize_payload(payload)
+
+        if message_bus_channels.include?(channel)
+          Mantle::MessageRouter.new(channel, message).route
+        end
+      end
+    end
+
+    def deserialize_payload(payload)
+      res = JSON.parse(payload)
+      [res.fetch("channel"), res.fetch("message")]
+    end
+
+    def hours_ago_in_seconds(hours)
+      hour_seconds = 60 * 60 * hours
+      Time.now.utc.to_f - hour_seconds
+    end
+
+    private
+
+    def serialize_payload(channel, message)
+      payload = { channel: channel, message: message }
+      JSON.generate(payload)
+    end
+  end
+end

data/lib/mantle/cli.rb

@@ -0,0 +1,73 @@
+require 'optparse'
+require 'fileutils'
+
+require 'mantle'
+
+module Mantle
+  class CLI
+
+    def initialize
+      @options = {}
+    end
+
+    def setup(args = ARGV)
+      parse_options(args)
+      load_config
+      configure_sidekiq
+    end
+
+    def parse_options(args)
+      optparser = OptionParser.new do |opts|
+        opts.banner = "Usage: mantle <command> [options]"
+
+        opts.on("-c", "--config CONFIG_FILE",
+                "Path to configuration file (initializer)") do |arg|
+          options[:config] = arg
+        end
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+
+        opts.on_tail("--version", "Show version") do
+          puts ::Mantle::VERSION
+          exit
+        end
+      end
+
+      optparser.parse!(args)
+    end
+
+    def load_config
+      if options[:config]
+        require File.expand_path(options[:config])
+      else
+        require File.expand_path("./config/initializers/mantle")
+      end
+    end
+
+    def configure_sidekiq
+      if namespace = Mantle.configuration.redis_namespace
+        Mantle.logger.info("Configuring Mantle to listen on Redis namespace: #{namespace}")
+
+        Sidekiq.configure_client do |config|
+          config.redis = { url: ENV["REDIS_URL"], namespace: namespace }
+        end
+
+        Sidekiq.configure_server do |config|
+          config.redis = { url: ENV["REDIS_URL"], namespace: namespace }
+        end
+      end
+    end
+
+    def listen
+      Mantle::MessageBus.new.listen
+    end
+
+    private
+
+    attr_reader :options
+  end
+end
+