chasqui 0.0.1 → 0.1.0

data/spec/integration/resque_spec.rb

@@ -0,0 +1,113 @@
+require 'spec_helper'
+
+describe "Chasqui resque integration" do
+
+  PUBLISH_EVENTS = [
+    { event: 'user.signup',    data: ['Kelly'] },
+    { event: 'account.credit', data: [1337, 'Kelly'] },
+    { event: 'account.debit',  data: [10, 'Kelly'] },
+    { event: 'user.signup',    data: ['Travis'] },
+    { event: 'account.debit',  data: [9000, 'Kelly'] },
+    { event: 'user.cancel',    data: ['Kelly'] },
+    { event: 'account.credit', data: [42, 'Travis'] },
+  ]
+
+  EXPECTED_EVENTS = {
+    'app1' => [
+      { event: 'user.signup', data: ['Kelly'] },
+      { event: 'user.signup', data: ['Travis'] },
+    ],
+    'app2' => [
+      { event: 'account.credit', data: [1337, 'Kelly'] },
+      { event: 'account.debit',  data: [10, 'Kelly'] },
+      { event: 'account.debit',  data: [9000, 'Kelly'] },
+      { event: 'user.cancel',    data: ['Kelly'] },
+      { event: 'account.credit', data: [42, 'Travis'] },
+    ],
+  }
+
+  before do
+    @subscriber_queues = %w(app1 app2)
+    @redis_url = 'redis://localhost:6379/13'
+    @redis = Redis.new url: @redis_url
+    @redis.keys('*').each { |k| @redis.del k }
+
+    Chasqui.configure do |c|
+      c.channel = 'integration'
+      c.redis = @redis_url
+    end
+
+    @pids = []
+    start_resque_workers
+
+    # Wait for subscribers to register before starting the broker so we don't miss events.
+    sleep 1
+
+    start_chasqui_broker
+  end
+
+  after do
+    terminate_child_processes
+  end
+
+  it 'works' do
+    PUBLISH_EVENTS.each do |event|
+      Chasqui.publish event[:event], *event[:data]
+    end
+
+    begin
+      Timeout::timeout(10) do
+        EXPECTED_EVENTS.each do |subscriber_queue, events|
+          events.each do |expected|
+            _, payload = @redis.blpop "chasqui:#{subscriber_queue}:event_log"
+            actual = JSON.parse payload
+            expect(actual['event']).to eq(expected[:event])
+            expect(actual['data']).to eq(expected[:data])
+          end
+        end
+      end
+    rescue TimeoutError
+      fail "Failed to process all events in a timely manner."
+    end
+  end
+
+  def start_chasqui_broker
+    @pids << fork do
+      exec './bin/chasqui',
+        '--logfile', 'tmp/resque-spec.log',
+        '--redis', @redis_url,
+        '--debug'
+    end
+  end
+
+  def start_resque_workers
+    @subscriber_queues.each do |queue|
+      @pids << fork do
+        ENV['CHASQUI_ENV'] = 'test'
+        ENV['QUEUE'] = queue
+        ENV['TERM_CHILD'] = '1'
+        ENV['INTERVAL'] = '0.1'
+        ENV['REDIS_NAMESPACE'] = "resque:#{queue}"
+        ENV['REDIS_URL'] = @redis_url
+        exec 'bundle', 'exec', 'rake', 'resque:work'
+      end
+    end
+  end
+
+  def terminate_child_processes
+    Timeout::timeout(10) do
+      @pids.each { |pid| kill 'TERM', pid rescue nil }
+    end
+  rescue TimeoutError
+    @pids.each { |pid| kill 'KILL', pid rescue nil }
+    fail "One or more child processes failed to terminate in a timely manner"
+  end
+
+  def kill(signal, pid)
+    Process.kill signal, pid
+    unless signal == 'KILL'
+      pid, status = Process.waitpid2 pid, 0
+      expect(status.exitstatus).to eq(0)
+    end
+  end
+end

data/spec/integration/subscribers.rb

@@ -0,0 +1,14 @@
+def log_event subscriber, args
+  event = subscriber.current_event
+  payload = { event: event['event'], data: args }.to_json
+  subscriber.redis.rpush "#{subscriber.queue}:event_log", payload
+end
+
+Chasqui.subscribe queue: 'app1', channel: 'integration' do
+  on('user.signup') { |*args| log_event self, args }
+end
+
+Chasqui.subscribe queue: 'app2', channel: 'integration' do
+  on('account.*')   { |*args| log_event self, args }
+  on('user.cancel') { |*args| log_event self, args }
+end

data/spec/lib/chasqui/broker_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe Chasqui::Broker do
+  let(:broker) { Chasqui::Broker.new }
+  before { reset_chasqui }
+
+  it 'forwards events to subscriber queues' do
+    expect(-> { broker.forward_event }).to raise_error(NotImplementedError)
+  end
+end

data/spec/lib/chasqui/cli_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+require 'chasqui/cli'
+
+describe Chasqui::CLI do
+
+  describe 'options' do
+    before { reset_config }
+
+    context do
+      before { Chasqui.config.broker_poll_interval = 1 }
+      after { File.unlink 'tmp/test.log' if File.exists? 'tmp/test.log' }
+
+      it 'changes the logfile' do
+        cli = Chasqui::CLI.new ['chasqui', '-f', 'tmp/test.log']
+        expect(cli.logfile).to eq('tmp/test.log')
+
+        cli.configure
+        Chasqui.logger.warn 'tmp/test.log message'
+        expect(File.read('tmp/test.log')).to match('tmp/test.log message')
+      end
+    end
+
+    it 'changes the redis connection' do
+        cli = Chasqui::CLI.new ['chasqui', '-r', 'redis://10.0.0.23:6379/5']
+        expect(cli.redis_url).to eq('redis://10.0.0.23:6379/5')
+
+        cli.configure
+        expect(Chasqui.redis.redis.client.host).to eq('10.0.0.23')
+    end
+
+    it 'changes the inbox queue' do
+        cli = Chasqui::CLI.new ['chasqui', '-q', 'inbox2']
+        expect(cli.inbox_queue).to eq('inbox2')
+
+        cli.configure
+        expect(Chasqui.inbox).to eq('inbox2')
+    end
+
+    it 'enables verbose mode (debug logging)' do
+        cli = Chasqui::CLI.new ['chasqui', '--debug']
+        expect(cli.debug).to be true
+
+        cli.configure
+        expect(Chasqui.logger.level).to eq(Logger::DEBUG)
+    end
+
+    it 'displays the version' do
+      cli = Chasqui::CLI.new ['chasqui', '-v']
+      cli.run
+      expect { cli.run }.to output(/chasqui #{Chasqui::VERSION}/).to_stdout
+    end
+
+    it 'displays help' do
+      cli = Chasqui::CLI.new ['chasqui', '-h', '--help']
+      expect { cli.run }.to output(/Usage: chasqui \[options\]/).to_stdout
+    end
+  end
+
+  describe '#run' do
+    before { reset_chasqui }
+
+    it 'configures and starts the broker' do
+      expect(Chasqui::Broker).to receive(:start)
+
+      argv = %w(chasqui -f tmp/test.log --debug -q inbox2 -r redis://127.0.0.1:6379/0)
+      Chasqui::CLI.new(argv).run
+
+      expect(Chasqui.logger.level).to eq(Logger::DEBUG)
+      expect(Chasqui.inbox_queue).to eq('inbox2')
+      expect(Chasqui.redis.client.host).to eq('127.0.0.1')
+    end
+  end
+
+end

data/spec/lib/chasqui/multi_broker_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+
+describe Chasqui::MultiBroker do
+  let(:broker) { Chasqui::MultiBroker.new }
+  before { reset_chasqui }
+
+  describe '#forward_event' do
+    before do
+      Chasqui.config.channel = 'app1'
+      Chasqui.subscribe queue: 'queue1', channel: 'app1'
+      Chasqui.subscribe queue: 'queue2', channel: 'app2'
+      Chasqui.subscribe queue: 'queue3', channel: 'app1'
+    end
+
+    it 'places the event on all subscriber queues' do
+      Chasqui.publish 'foo.bar', 'A'
+      broker.forward_event
+
+      expect(redis.llen('inbox')).to eq(0)
+
+      expect(redis.llen('queue:queue1')).to eq(1)
+      expect(redis.llen('queue:queue2')).to eq(0)
+      expect(redis.llen('queue:queue3')).to eq(1)
+
+      event = { 'event' => 'foo.bar', 'channel' => 'app1', 'data' => ['A'] }
+
+      job1 = JSON.parse redis.rpop('queue:queue1')
+      expect(job1['args']).to include(event)
+
+      job3 = JSON.parse redis.rpop('queue:queue3')
+      expect(job3['args']).to include(event)
+    end
+
+    it 'blocks on empty queues' do
+      thread = Thread.new { broker.forward_event }
+
+      begin
+        Timeout::timeout(1) do
+          Chasqui.config.redis = Redis.new
+          Chasqui.config.channel = 'app2'
+          Chasqui.publish 'foo.bar', 'A'
+
+          job = JSON.parse redis.brpop('queue:queue2')[1]
+          expect(job).to include('class' => 'Chasqui::Subscriber__queue2')
+          expect(job).to include('args' =>
+            [{ 'event' => 'foo.bar', 'channel' => 'app2', 'data' => ['A'] }])
+        end
+      ensure
+        thread.kill
+      end
+    end
+
+    it "doesn't lose events if the broker fails" do
+      Chasqui.config.channel = 'app2'
+      Chasqui.publish 'foo', 'process'
+      Chasqui.publish 'foo', 'keep in queue'
+      allow(broker.redis).to receive(:smembers).and_raise(Redis::ConnectionError)
+
+      expect(-> { broker.forward_event }).to raise_error(Redis::ConnectionError)
+      expect(redis.llen('queue:queue2')).to eq(0)
+      expect(redis.llen(broker.in_progress_queue)).to eq(1)
+
+      allow(broker.redis).to receive(:smembers).and_call_original
+      broker.forward_event
+      expect(redis.llen('queue:queue2')).to eq(1)
+
+      job = JSON.parse redis.rpop('queue:queue2')
+      expect(job['args']).to include(
+        'event' => 'foo', 'channel' => 'app2', 'data' => ['process'])
+      expect(redis.llen(broker.in_progress_queue)).to eq(0)
+    end
+  end
+end

data/spec/lib/chasqui/subscriber_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper'
+
+describe Chasqui::Subscriber do
+  let(:subscriber) { Chasqui::Subscriber.new 'my-queue', 'my.channel' }
+
+  it { expect(subscriber.queue).to eq('my-queue') }
+  it { expect(subscriber.channel).to eq('my.channel') }
+
+  describe '#on' do
+    it 'registers the event handlers' do
+      subscriber.on('foo') { |foo| foo }
+      subscriber.on('zig.zag') { |*args| args }
+
+      pattern = subscriber.matching_handler_patterns_for('foo').first
+      expect(subscriber.call_handler(pattern, 'bar')).to eq('bar')
+
+      pattern = subscriber.matching_handler_patterns_for('zig.zag').first
+      expect(subscriber.call_handler(pattern, 1, 2, 3, 4)).to eq([1, 2, 3, 4])
+    end
+
+    it 'raises when registering duplicate handlers' do
+      subscriber.on('foo') { |foo| foo }
+      expect(-> {
+        subscriber.on('foo') { |a, b| a + b }
+      }).to raise_error(Chasqui::HandlerAlreadyRegistered)
+    end
+  end
+
+  describe '#matching_handler_patterns_for' do
+    it 'always returns an array' do
+      expect(subscriber.matching_handler_patterns_for('unknown')).to eq([])
+    end
+
+    it 'matches single event' do
+      p = Proc.new { }
+      subscriber.on('foo', &p)
+      expect(subscriber.matching_handler_patterns_for('foo')).to eq([/\Afoo\z/])
+    end
+
+    it 'matches wildcards' do
+      p = 6.times.map { Proc.new { } }
+      subscriber.on('foo*', &p[0])
+      subscriber.on('bar', &p[1])
+      subscriber.on('*bar', &p[2])
+      subscriber.on('*', &p[3])
+      subscriber.on('*a*', &p[4])
+      subscriber.on('*z*', &p[5])
+      expect(subscriber.matching_handler_patterns_for('foo.bar')).to eq([
+        /\Afoo.*\z/,
+        /\A.*bar\z/,
+        /\A.*\z/,
+        /\A.*a.*\z/
+      ])
+    end
+  end
+
+  describe '#perform' do
+    it 'calls the matching event handlers' do
+      calls = []
+      subscriber.on('foo.bar') { |a, b| calls << a + b }
+      subscriber.on('foo.*') { |a, b| calls << a ** b }
+      subscriber.perform(redis, { 'event' => 'foo.bar', 'data' => [3, 4] })
+      expect(calls.sort).to eq([7, 81])
+    end
+  end
+
+end

data/spec/lib/chasqui/workers/resque_worker_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+require 'resque'
+
+describe Chasqui::ResqueWorker do
+  let(:subscriber) { FakeSubscriber.new 'my-queue', 'my.channel'}
+
+  describe '.create' do
+    it 'configures a new worker' do
+      worker_class = Chasqui::ResqueWorker.create(subscriber)
+      expect(worker_class.name).to eq('Chasqui::Subscriber__my_queue')
+      expect(worker_class.ancestors).to include(Chasqui::ResqueWorker)
+      expect(worker_class.instance_variable_get(:@queue)).to eq('my-queue')
+    end
+  end
+
+  describe '.perform' do
+    let(:worker) { Chasqui::ResqueWorker.create(subscriber) }
+
+    it 'delegates to the subscriber' do
+      event = { 'event' => 'foo', 'data' => ['bar'] }
+      worker.perform event
+      received_event = subscriber.events.shift
+      expect(received_event).to eq(event)
+    end
+  end
+
+end

data/spec/lib/chasqui/workers/sidekiq_worker_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+if sidekiq_supported_ruby_version?
+
+  describe Chasqui::SidekiqWorker do
+    let(:subscriber) { FakeSubscriber.new 'my-queue', 'my.channel'}
+
+    describe '.create' do
+      it 'configures a new worker' do
+        worker_class = Chasqui::SidekiqWorker.create(subscriber)
+        expect(worker_class.included_modules).to include(Sidekiq::Worker)
+        expect(worker_class.sidekiq_options).to include('queue' => 'my-queue')
+        expect(worker_class.new).to be_kind_of(Chasqui::SidekiqWorker)
+      end
+    end
+
+    describe '#perform' do
+      let(:worker_class) { Chasqui::SidekiqWorker.create(subscriber) }
+
+      it 'delegates to the subscriber' do
+        event = { 'event' => 'foo', 'data' => ['bar'] }
+        worker = worker_class.new
+        worker.perform event
+        received_event = subscriber.events.shift
+        expect(received_event).to eq(event)
+      end
+    end
+  end
+
+end

data/spec/lib/chasqui_spec.rb

@@ -0,0 +1,185 @@
+require 'spec_helper'
+
+describe Chasqui do
+  it 'has a version number' do
+    expect(Chasqui::VERSION).not_to be nil
+  end
+
+  describe '.configure' do
+    before { reset_config }
+
+    context 'defaults' do
+      it { expect(Chasqui.channel).to eq('__default') }
+      it { expect(Chasqui.inbox_queue).to eq('inbox') }
+      it { expect(Chasqui.redis.client.db).to eq(0) }
+      it { expect(Chasqui.config.broker_poll_interval).to eq(3) }
+      it { expect(Chasqui.config.worker_backend).to eq(nil) }
+
+      it do
+        # remove chasqui's test environment logger
+        Chasqui.config[:logger] = nil
+        expect(Chasqui.logger).to be_kind_of(Logger)
+      end
+
+      it { expect(Chasqui.logger.level).to eq(Logger::INFO) }
+      it { expect(Chasqui.logger.progname).to eq('chasqui') }
+    end
+
+    it 'configures the channel' do
+      Chasqui.config.channel = 'com.example.test'
+      expect(Chasqui.channel).to eq('com.example.test')
+    end
+
+    it 'accepts a block' do
+      Chasqui.configure { |config| config.channel = 'com.example.test' }
+      expect(Chasqui.channel).to eq('com.example.test')
+    end
+
+    it 'configures the inbox queue' do
+      Chasqui.config.inbox_queue = 'foo'
+      expect(Chasqui.inbox).to eq('foo')
+    end
+
+    it 'configures the broker poll interval' do
+      Chasqui.config.broker_poll_interval = 1
+      expect(Chasqui.config.broker_poll_interval).to eq(1)
+    end
+
+    context 'redis' do
+      it 'accepts config options' do
+        redis_config = { host: '10.0.3.24' }
+        Chasqui.config.redis = redis_config
+        expect(Chasqui.redis.client.host).to eq('10.0.3.24')
+      end
+
+      it 'accepts an initialized client' do
+        redis = Redis.new db: 2
+        Chasqui.config.redis = redis
+        expect(Chasqui.redis.client.db).to eq(2)
+      end
+
+      it 'accepts URLs' do
+        Chasqui.config.redis = 'redis://10.0.1.21:12345/0'
+        expect(Chasqui.redis.client.host).to eq('10.0.1.21')
+      end
+
+      it 'uses a namespace' do
+        Chasqui.redis.set 'foo', 'bar'
+        expect(Chasqui.redis.redis.get 'chasqui:foo').to eq('bar')
+      end
+    end
+
+    describe 'logger' do
+      it 'accepts a log device' do
+        logs = StringIO.new
+        Chasqui.config.logger = logs
+        Chasqui.logger.info "status"
+        Chasqui.logger.warn "error"
+
+        logs.rewind
+        output = logs.read
+
+        %w(chasqui INFO status WARN error).each do |text|
+          expect(output).to match(text)
+        end
+      end
+
+      it 'accepts a logger-like object' do
+        fake_logger = FakeLogger.new
+        Chasqui.config.logger = fake_logger
+        expect(Chasqui.logger).to eq(fake_logger)
+        expect(Chasqui.logger.progname).to eq('chasqui')
+      end
+    end
+  end
+
+  describe '.publish' do
+    before { reset_chasqui }
+
+    it 'pushes messages to the inbox queue' do
+      payloads = [
+        [1, 2, {'foo'=>'bar'}],
+        [3, 4, {'biz'=>'baz'}]
+      ]
+
+      payloads.each do |args|
+        Chasqui.publish 'test.event', *args
+      end
+
+      payloads.each do |data|
+        event = JSON.load Chasqui.redis.rpop('inbox')
+        expect(event['event']).to eq('test.event')
+        expect(event['channel']).to eq('__default')
+        expect(event['data']).to eq(data)
+      end
+    end
+
+    it 'supports channels' do
+      Chasqui.config.channel = 'my.app'
+      Chasqui.publish 'test.event', :foo
+      event = JSON.load Chasqui.redis.rpop('inbox')
+      expect(event['event']).to eq('test.event')
+      expect(event['channel']).to eq('my.app')
+      expect(event['data']).to eq(['foo'])
+    end
+  end
+
+  describe '.subscribe' do
+    before { reset_chasqui }
+
+    it 'saves subscriptions' do
+      sub1 = Chasqui.subscribe queue: 'app1-queue', channel: 'com.example.admin'
+      sub2 = Chasqui.subscribe queue: 'app2-queue', channel: 'com.example.admin'
+      sub3 = Chasqui.subscribe queue: 'app1-queue', channel: 'com.example.video'
+
+      queues = Chasqui.redis.smembers "subscribers:com.example.admin"
+      expect(queues.sort).to eq(['app1-queue', 'app2-queue'])
+
+      queues = Chasqui.redis.smembers "subscribers:com.example.video"
+      expect(queues).to eq(['app1-queue'])
+
+      expect(Chasqui.subscriber('app1-queue')).to eq(sub1)
+      expect(Chasqui.subscriber('app2-queue')).to eq(sub2)
+      expect(sub1).to eq(sub3)
+    end
+
+    it 'returns a subscriber' do
+      subscriber = Chasqui.subscribe queue: 'app1-queue', channel: 'com.example.admin'
+      expect(subscriber).to be_kind_of(Chasqui::Subscriber)
+    end
+
+    it 'yields a subscriber configuration context' do
+      $context = nil
+      Chasqui.subscribe queue: 'foo', channel: 'bar' do
+        $context = self
+      end
+      expect($context).to be_kind_of(Chasqui::Subscriber)
+    end
+  end
+
+  describe '.create_worker' do
+    let(:subscriber) { j }
+
+    it 'raises when no worker backend configured' do
+      expect(-> {
+        Chasqui.create_worker nil
+      }).to raise_error(Chasqui::ConfigurationError)
+    end
+
+    it 'creates a resque worker' do
+      subscriber = FakeSubscriber.new 'resque-queue', 'fake-channel'
+      Chasqui.config.worker_backend = :resque
+      worker = Chasqui.create_worker subscriber
+      expect(worker.new).to be_kind_of(Chasqui::ResqueWorker)
+    end
+
+    if sidekiq_supported_ruby_version?
+      it 'creates a sidekiq worker' do
+        subscriber = FakeSubscriber.new 'sidekiq-queue', 'fake-channel'
+        Chasqui.config.worker_backend = :sidekiq
+        worker = Chasqui.create_worker subscriber
+        expect(worker.new).to be_kind_of(Chasqui::SidekiqWorker)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,23 +1,10 @@
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'chasqui'
+require 'resque'
+require 'pp'
 
-class FakeRedis
-  def initialize
-    @queues = {}
-  end
+Dir['spec/support/*.rb'].each { |file| require File.expand_path(file) }
 
-  def rpush(key, value)
-    queue(key) << value
-  end
-
-  def lpop(key)
-    queue(key).shift
-  end
-
-  private
-
-  def queue(key)
-    @queues[key] ||= []
-    @queues[key]
-  end
+RSpec.configure do |c|
+  c.include ChasquiSpecHelpers
 end

data/spec/support/chasqui_spec_helpers.rb

@@ -0,0 +1,21 @@
+module ChasquiSpecHelpers
+
+  def reset_chasqui
+    reset_config
+    flush_redis
+  end
+
+  def reset_config
+    Chasqui.instance_variable_set(:@config, nil)
+    Chasqui.config.logger = open('/dev/null', 'w+')
+  end
+
+  def redis
+    Chasqui.redis
+  end
+
+  def flush_redis
+    redis.keys('*').each { |k| redis.del k }
+  end
+
+end

data/spec/support/fake_logger.rb

@@ -0,0 +1,5 @@
+class FakeLogger
+  attr_accessor :progname
+  def info(*args, &block)
+  end
+end

data/spec/support/fake_subscriber.rb

@@ -0,0 +1,12 @@
+class FakeSubscriber < Chasqui::Subscriber
+  attr_reader :events
+
+  def initialize(queue, channel)
+    super
+    @events ||= []
+  end
+
+  def perform(redis, event)
+    @events << event
+  end
+end

data/spec/support/sidekiq_compatibility_check.rb

@@ -0,0 +1,9 @@
+# Newer versions of sidekiq only support newer versions of ruby
+# https://github.com/mperham/sidekiq/blob/master/Changes.md#322
+def sidekiq_supported_ruby_version?
+  Gem::Version.new(RUBY_VERSION.dup) > Gem::Version.new('1.9.3')
+end
+
+if sidekiq_supported_ruby_version?
+  require 'sidekiq'
+end