chasqui 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7dc418405a5b23f20b67dc597c78926ee37f2f97
-  data.tar.gz: 8dbef72f6a1fe7546c3fa770d461d931ee396312
+  metadata.gz: 1bb216d9f86e895b3a2c48b686a9d3d92f991cf8
+  data.tar.gz: e736ccd90ee44c1aa827c4d59e09136ea63f9958
 SHA512:
-  metadata.gz: 6619038c7980b5bdb191014f3e14a3753c5a0256eeaa844035895fc23422736a3584772c8c77dd6b6eb4437ac5bbad019c79138ce0fdab9e8120f8069f03278d
-  data.tar.gz: f6b7da7f471ce28f665f44cfcd82edcf1702bb2de0d29dcebfc83baf4662ccfa5cbbbff5aed883b550b554a0d4702026ec42b7f65e713bfc022ecb2cc376b8fa
+  metadata.gz: d174c6cb4f929a2135f86fe7bf4c06e32d8f66088e25d18fa4048b872aaaa9430888f7cb5f901665dff430064dd24772164fb1b19aa6b48f3b3e74f7fa9d14a2
+  data.tar.gz: e4e64b8ee8453138def77a8f229e3fb002f0e794d2c0e0013db5eaa12bdd6bae46e109d707e95b024a6dae4081dbdcfcf2ff7288ea4ea5c9f8ce4ba55eb19532

data/.travis.yml

@@ -1,3 +1,6 @@
 language: ruby
+services:
+  - redis-server
 rvm:
-  - 2.1.3
+  - 2.2.2
+  - 1.9.3

data/Guardfile

@@ -0,0 +1,14 @@
+guard :rspec, cmd: "bundle exec rspec" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+end

data/README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/jbgo/chasqui.svg?branch=master)](https://travis-ci.org/jbgo/chasqui)
+
 # Chasqui
 
 Chasqui is a simple, lightweight, persistent implementation of the publish-subscribe (pub/sub)
@@ -68,12 +70,12 @@ Publishing events is simple.
 Chasqui.publish 'user.sign-up', user_id
 ```
 
-To prevent conflicts with other applications, you can choose a unique namespace for your events.
+To prevent conflicts with other applications, you can choose a unique channel for your events.
 
 ```rb
 # config/initializers/chasqui.rb
 Chasqui.configure do |config|
-  config.namespace = 'com.example.myapp'
+  config.channel = 'com.example.myapp'
 end
 ```
 
@@ -82,11 +84,9 @@ Now when you call `Chasqui.publish 'event.name', data, ...`, Chasqui will publis
 
 ## Subscribing to events
 
-__NOT IMPLMENTED YET__
-
 ```rb
 # file: otherapp/app/subscribers/user_events.rb
-Chasqui.subscribe queue: 'unique_queue_name_for_app', namespace: 'com.example.myapp' do
+Chasqui.subscribe queue: 'unique_queue_name_for_app', channel: 'com.example.myapp' do
 
   on 'user.sign-up' do |user_id|
     user = User.find user_id
@@ -106,7 +106,7 @@ end
 
 ```rb
 Chasqui.configure do |config|
-  config.namespace = 'com.example.transcoder'
+  config.channel = 'com.example.transcoder'
   config.redis = ENV.fetch('REDIS_URL')
   config.workers = :sidekiq # or :resque
   ...

data/Rakefile

@@ -1,7 +1,23 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
+require 'resque/tasks'
+task 'resque:setup' => :environment
+
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
 
+task :environment do
+  $LOAD_PATH.unshift './lib'
+  require 'bundler/setup'
+  require 'chasqui'
+
+  if ENV['CHASQUI_ENV'] == 'test'
+    require './spec/integration/subscribers'
+  end
+
+  require 'resque'
+  Resque.redis = ENV['REDIS_URL'] if ENV['REDIS_URL']
+  Resque.redis.namespace = 'chasqui'
+end

data/bin/chasqui

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'chasqui/cli'
+
+Chasqui::CLI.new(ARGV).run

data/chasqui.gemspec

@@ -17,7 +17,13 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "redis"
+  spec.add_dependency "redis-namespace"
+
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "resque"
+  spec.add_development_dependency "sidekiq"
 end

data/examples/full.rb

@@ -1,6 +1,6 @@
 # file: admin/config/initializers/chasqui.rb
 Chasqui.configure do |config|
-  config.namespace = 'com.example.admin'
+  config.channel = 'com.example.admin'
   config.redis = ENV.fetch('REDIS_URL')
 end
 
@@ -28,7 +28,7 @@ Chasqui.configure do |config|
 end
 
 # file: transcoder/app/subscribers/video_subscriber.rb
-Chasqui.subscribe queue: 'transcoder.video', namespace: 'com.example.admin' do
+Chasqui.subscribe queue: 'transcoder.video', channel: 'com.example.admin' do
   on 'video.upload' do |video_id|
     begin
       Transcorder.transcode video_url(video_id)
@@ -41,14 +41,14 @@ Chasqui.subscribe queue: 'transcoder.video', namespace: 'com.example.admin' do
 end
 
 # file: admin/app/subscribers/video_subscriber.rb
-Chasqui.subscribe queue: 'admin.events', namespace: 'com.example.transcoder' do
+Chasqui.subscribe queue: 'admin.events', channel: 'com.example.transcoder' do
 
   on 'transcoder.video.complete' do |video_id|
     video = Video.find video_id
     VideoMailer.transcode_complete(video).deliver
   end
 
-  on 'transcoder.video.complete' do |video_id, error|
+  on 'transcoder.video.error' do |video_id, error|
     video = Video.find video_id
     VideoMailer.transcode_error(video, error).deliver
   end

data/lib/chasqui/broker.rb

@@ -0,0 +1,70 @@
+require 'timeout'
+
+class Chasqui::Broker
+  attr_reader :config
+
+  extend Forwardable
+  def_delegators :@config, :redis, :inbox, :logger
+
+  ShutdownSignals = %w(INT QUIT ABRT TERM).freeze
+
+  # To prevent unsuspecting clients from blocking forever, the broker uses
+  # it's own private redis connection.
+  def initialize
+    @shutdown_requested = nil
+    @config = Chasqui.config.dup
+    @config.redis = Redis.new @config.redis.client.options
+  end
+
+  def start
+    install_signal_handlers
+
+    logger.info "broker started with pid #{Process.pid}"
+    logger.info "configured to fetch events from #{inbox} on #{redis.inspect}"
+
+    until_shutdown_requested { forward_event }
+  end
+
+  def forward_event
+    raise NotImplementedError.new "please define #forward_event in a subclass of #{self.class.name}"
+  end
+
+  class << self
+    def start
+      Chasqui::MultiBroker.new.start
+    end
+  end
+
+  private
+
+  def install_signal_handlers
+    ShutdownSignals.each do |signal|
+      trap(signal) { @shutdown_requested = signal }
+    end
+  end
+
+  def until_shutdown_requested
+    catch :shutdown do
+      loop do
+        with_timeout do
+          if @shutdown_requested
+            logger.info "broker received signal, #@shutdown_requested. shutting down"
+            throw :shutdown
+          else
+            yield
+          end
+        end
+      end
+    end
+  end
+
+  def with_timeout
+    # This timeout is a failsafe for an improperly configured broker
+    Timeout::timeout(config.broker_poll_interval + 1) do
+      yield
+    end
+  rescue TimeoutError
+    logger.warn "broker poll interval exceeded for broker, #{self.class.name}"
+  end
+
+end

data/lib/chasqui/cli.rb

@@ -0,0 +1,78 @@
+require 'optparse'
+require 'ostruct'
+require 'chasqui'
+
+class Chasqui::CLI
+  extend Forwardable
+  def_delegators :@options, :logfile, :redis_url, :inbox_queue, :debug, :version
+
+  def initialize(argv)
+    build_options(argv)
+  end
+
+  def configure
+    Chasqui.configure do |config|
+      config.logger = options.logfile if options.logfile
+      config.redis = options.redis_url if options.redis_url
+      config.inbox_queue = options.inbox_queue if options.inbox_queue
+      config.logger.level = Logger::DEBUG if options.debug
+    end
+  end
+
+  def run
+    configure
+
+    if options.version
+      puts "chasqui #{Chasqui::VERSION}"
+    elsif options.help
+      puts @parser.help()
+    else
+      Chasqui::Broker.start
+    end
+  rescue => ex
+    Chasqui.logger.fatal ex.inspect
+    Chasqui.logger.fatal ex.backtrace.join("\n")
+  end
+
+  private
+
+  def options
+    @options ||= build_options
+  end
+
+  def build_options(argv)
+    opts = {}
+
+    @parser = OptionParser.new do |o|
+      o.banner = "Usage: #{argv[0]} [options]"
+
+      o.on('-f', '--logfile PATH', 'log file path') do |arg|
+        opts[:logfile] = arg
+      end
+
+      o.on('-r', '--redis URL', 'redis connection URL') do |arg|
+        opts[:redis_url] = arg
+      end
+
+      o.on('-q', '--inbox-queue NAME', 'name of the queue from which chasqui broker consumes events') do |arg|
+        opts[:inbox_queue] = arg
+      end
+
+      o.on('-d', '--debug', 'enable debug logging') do |arg|
+        opts[:debug] = arg
+      end
+
+      o.on('-v', '--version', 'show version and exit') do |arg|
+        opts[:version] = arg
+      end
+
+      o.on('-h', '--help', 'show this help pessage') do |arg|
+        opts[:help] = arg
+      end
+    end
+
+    @parser.parse!(argv)
+    @options = OpenStruct.new opts
+  end
+
+end

data/lib/chasqui/config.rb

@@ -0,0 +1,78 @@
+module Chasqui
+
+  Defaults = {
+    inbox_queue: 'inbox',
+    redis_namespace: 'chasqui',
+    publish_channel: '__default',
+    broker_poll_interval: 3
+  }.freeze
+
+  class ConfigurationError < StandardError
+  end
+
+  CONFIG_SETTINGS = [
+    :broker_poll_interval,
+    :channel,
+    :inbox_queue,
+    :logger,
+    :redis,
+    :worker_backend
+  ]
+
+  class Config < Struct.new(*CONFIG_SETTINGS)
+    def channel
+      self[:channel] ||= Defaults.fetch(:publish_channel)
+    end
+
+    def inbox_queue
+      self[:inbox_queue] ||= Defaults.fetch(:inbox_queue)
+    end
+    alias inbox inbox_queue
+
+    def redis
+      unless self[:redis]
+        self.redis = Redis.new
+      end
+
+      self[:redis]
+    end
+
+    def redis=(redis_config)
+      client = case redis_config
+      when Redis
+        redis_config
+      when String
+        Redis.new url: redis_config
+      else
+        Redis.new redis_config
+      end
+
+      self[:redis] = Redis::Namespace.new(Defaults.fetch(:redis_namespace), redis: client)
+    end
+
+    def logger
+      unless self[:logger]
+        self.logger = STDOUT
+      end
+
+      self[:logger]
+    end
+
+    def logger=(new_logger)
+      lg = if new_logger.respond_to? :info
+        new_logger
+      else
+        Logger.new(new_logger).tap do |lg|
+          lg.level = Logger::INFO
+        end
+      end
+
+      lg.progname = 'chasqui'
+      self[:logger] = lg
+    end
+
+    def broker_poll_interval
+      self[:broker_poll_interval] ||= Defaults.fetch(:broker_poll_interval)
+    end
+  end
+end

data/lib/chasqui/multi_broker.rb

@@ -0,0 +1,56 @@
+class Chasqui::MultiBroker < Chasqui::Broker
+
+  def forward_event
+    event = receive or return
+    queues = subscriber_queues event
+
+    redis.multi do
+      queues.each do |queue|
+        dispatch event, queue
+      end
+      redis.rpop(in_progress_queue)
+    end
+
+    logger.info "processed event: #{event['event']}, on channel: #{event['channel']}"
+  end
+
+  def in_progress_queue
+    "#{inbox}:in_progress"
+  end
+
+  private
+
+  def receive
+    event = retry_failed_event || dequeue
+    logger.debug "received event: #{event['event']}, on channel: #{event['channel']}"
+
+    JSON.parse event
+  end
+
+  def retry_failed_event
+    redis.lrange(in_progress_queue, -1, -1).first.tap do |event|
+      unless event.nil?
+        logger.warn "detected failed event delivery, attempting recovery"
+      end
+    end
+  end
+
+  def dequeue
+    redis.brpoplpush(inbox, in_progress_queue, timeout: config.broker_poll_interval).tap do |event|
+      if event.nil?
+        logger.debug "reached timeout for broker poll interval: #{config.broker_poll_interval} seconds"
+      end
+    end
+  end
+
+  def dispatch(event, queue)
+    job = { class: "Chasqui::Subscriber__#{queue}", args: [event] }.to_json
+    logger.debug "dispatching event to queue: #{queue}, with job: #{job}"
+    redis.rpush "queue:#{queue}", job
+  end
+
+  def subscriber_queues(event)
+    redis.smembers "subscribers:#{event['channel']}"
+  end
+
+end

data/lib/chasqui/subscriber.rb

@@ -0,0 +1,74 @@
+require 'set'
+
+module Chasqui
+
+  HandlerAlreadyRegistered = Class.new StandardError
+
+  class Subscriber
+    attr_accessor :redis, :current_event
+    attr_reader :queue, :channel
+
+    def initialize(queue, channel)
+      @queue = queue
+      @channel = channel
+    end
+
+    def on(event_name, &block)
+      pattern = pattern_for_event event_name
+
+      if handler_patterns.include? pattern
+        raise HandlerAlreadyRegistered.new "handler already registered for event: #{event_name}"
+      else
+        handler_patterns << pattern
+        define_handler_method pattern, &block
+      end
+    end
+
+    def perform(redis_for_worker, event)
+      self.redis = redis_for_worker
+      self.current_event = event
+
+      matching_handler_patterns_for(event['event']).each do |pattern|
+        call_handler pattern, *event['data']
+      end
+    end
+
+    def matching_handler_patterns_for(event_name)
+      handler_patterns.select do |pattern|
+        pattern =~ event_name
+      end
+    end
+
+    def call_handler(pattern, *args)
+      send "handler__#{pattern.to_s}", *args
+    end
+
+    def evaluate(&block)
+      @self_before_instance_eval = eval "self", block.binding
+      instance_eval &block
+    end
+
+    private
+
+    def handler_patterns
+      @handler_patterns ||= Set.new
+    end
+
+    def method_missing(method, *args, &block)
+      if @self_before_instance_eval
+        @self_before_instance_eval.send method, *args, &block
+      else
+        super
+      end
+    end
+
+    def pattern_for_event(event_name)
+      /\A#{event_name.to_s.downcase.gsub('*', '.*')}\z/
+    end
+
+    def define_handler_method(pattern, &block)
+      self.class.send :define_method, "handler__#{pattern.to_s}", &block
+    end
+
+  end
+end

data/lib/chasqui/version.rb

@@ -1,3 +1,3 @@
 module Chasqui
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/lib/chasqui/workers/resque_worker.rb

@@ -0,0 +1,40 @@
+class Chasqui::ResqueWorker
+
+  class << self
+
+    # Factory method to create a Resque worker class for a Chasqui::Subscriber instance.
+    def create(subscriber)
+      find_or_build_worker(subscriber).tap do |worker|
+        worker.class_eval do
+          @queue = subscriber.queue
+          @subscriber = subscriber
+
+          def self.perform(event)
+            @subscriber.perform Resque.redis, event
+          end
+        end
+      end
+    end
+
+    private
+
+    def find_or_build_worker(subscriber)
+      class_name = class_name_for subscriber
+
+      if Chasqui.const_defined? class_name
+        Chasqui.const_get class_name
+      else
+        Class.new(Chasqui::ResqueWorker).tap do |worker|
+          Chasqui.const_set class_name, worker
+        end
+      end
+    end
+
+    def class_name_for(subscriber)
+      queue_name_constant = subscriber.queue.gsub(/[^\w]/, '_')
+      "Subscriber__#{queue_name_constant}".to_sym
+    end
+
+  end
+
+end

data/lib/chasqui/workers/sidekiq_worker.rb

@@ -0,0 +1,25 @@
+class Chasqui::SidekiqWorker
+
+  class << self
+
+    def create(subscriber)
+      Class.new(Chasqui::SidekiqWorker) do
+        include Sidekiq::Worker
+        sidekiq_options queue: subscriber.queue
+        @subscriber = subscriber
+
+        def perform(event)
+          self.class.subscriber.perform "TODO: redis", event
+        end
+
+        private
+
+        def self.subscriber
+          @subscriber
+        end
+      end
+    end
+
+  end
+
+end

data/lib/chasqui.rb

@@ -1,39 +1,77 @@
-require 'ostruct'
+require 'forwardable'
 require 'json'
+require 'logger'
+require 'redis'
+require 'redis-namespace'
+
 require "chasqui/version"
+require "chasqui/config"
+require "chasqui/broker"
+require "chasqui/multi_broker"
+require "chasqui/subscriber"
+require "chasqui/workers/resque_worker"
+require "chasqui/workers/sidekiq_worker"
 
 module Chasqui
+  class << self
+    extend Forwardable
+    def_delegators :config, :redis, :channel, :inbox, :inbox_queue, :logger
 
-  Config = Struct.new :namespace, :redis
+    def configure(&block)
+      @config ||= Config.new
+      yield @config
+    end
 
-  Defaults = {
-    publish_queue: 'chasqui.inbox'
-  }.freeze
+    def config
+      @config ||= Config.new
+    end
 
-  module ClassMethods
-    def namespace
-      @config.namespace unless @config.nil?
+    def publish(event, *args)
+      payload = { event: event, channel: channel, data: args }
+      redis.lpush inbox_queue, payload.to_json
     end
 
-    def redis
-      @config.redis unless @config.nil?
+    def subscribe(options={}, &block)
+      queue = options.fetch :queue
+      channel = options.fetch :channel
+
+      register_subscriber(queue, channel).tap do |sub|
+        sub.evaluate(&block) if block_given?
+        Chasqui::ResqueWorker.create sub
+        redis.sadd "subscribers:#{channel}", queue
+      end
     end
 
-    def publish_queue
-      Defaults[:publish_queue]
+    def subscriber(queue)
+      subscribers[queue.to_s]
     end
 
-    def configure(&block)
-      @config ||= Config.new
-      yield @config
+    def create_worker(subscriber)
+      case config.worker_backend
+      when :resque
+        Chasqui::ResqueWorker.create subscriber
+      when :sidekiq
+        Chasqui::SidekiqWorker.create subscriber
+      else
+        raise ConfigurationError.new(
+          "Please choose a supported worker_backend. Choices: #{supported_worker_backends}")
+      end
     end
 
-    def publish(event, *args)
-      name = namespace ? "#{namespace}.#{event}" : event
-      redis.rpush publish_queue, { name: name, data: args }.to_json
+    private
+
+    def register_subscriber(queue, channel)
+      subscribers[queue.to_s] ||= Subscriber.new queue, channel
     end
-  end
 
-end
+    def subscribers
+      @subscribers ||= {}
+    end
+
+    SUPPORTED_WORKER_BACKENDS = [:resque, :sidekiq].freeze
 
-Chasqui.extend Chasqui::ClassMethods
+    def supported_worker_backends
+      SUPPORTED_WORKER_BACKENDS.join(', ')
+    end
+  end
+end