wide_receiver 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9f973638dac7294b4f2bfc94c2fc3c0d372a27c7
+  data.tar.gz: f57c4dafbc355f6a41bb6bedf0b861c33e1a2a4a
+SHA512:
+  metadata.gz: d726e625d0fa7564265191f547c361d4ca5eba73197d7322640ea2a110bb6114a5291b61853833b1b78628b6c8d310b86091e9b37e6ab283c2b2f0fddb9df237
+  data.tar.gz: 3a2761f89de4cb934936e8b7ef5ac443fbdfd96c36e4f5ab251f3ec12cf66d05ad1c9d607a0f415eeb3f8e5b5173322611597a01c9ec23b855cfd205f3780d30

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+bin
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/bundle

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.0
+  - 2.1.1
+  - rbx-2.1.1
+services:
+  - redis-server

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in wide_receiver.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Solomon White
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Wide Receiver
+
+Wide Receiver is a work queue system in which the worker classes decide which
+messages to process.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'wide_receiver'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install wide_receiver
+
+## Usage
+
+Configure Wide Receiver to connect to your queuing system (currently only
+supports Redis pubsub):
+
+```ruby
+WideReceiver::Config.instance.queue_url = 'redis://localhost/6379/5'
+```
+
+Create a worker class that extends `WideReceiver::Worker` and specifies which channel
+patterns it wants to process:
+
+```ruby
+class Rice
+  extend WideReceiver::Worker
+  listen 'montana.pass.*'
+
+  def perform(channel, message)
+    # do something with message
+  end
+end
+```
+
+Start the master process:
+
+```ruby
+WideReceiver::Master.new.start
+```
+
+This will start one thread per unique channel pattern.  When a message is
+received on a matching channel, each worker class that expressed interest in
+that pattern will be instantiated, and the instance will receive the message.
+
+## TODO
+
+- add support for RabbitMQ
+- add proper command line runner
+
+## Contributing
+
+1. Fork it ( http://github.com/rubysolo/wide_receiver/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+desc "Run specs"
+task :spec do
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.rspec_opts = %w{--colour --format progress}
+    t.pattern = 'spec/*_spec.rb'
+  end
+end
+
+desc "Default: run specs."
+task default: :spec

data/lib/wide_receiver.rb

@@ -0,0 +1,5 @@
+require "wide_receiver/config"
+require "wide_receiver/registry"
+require "wide_receiver/worker"
+require "wide_receiver/master"
+require "wide_receiver/version"

data/lib/wide_receiver/adapters.rb

@@ -0,0 +1,10 @@
+require 'multi_json'
+
+module WideReceiver
+  module Adapters
+  end
+end
+
+Dir[File.dirname(__FILE__) + '/adapters/*.rb'].each do |adapter|
+  require_relative "adapters/#{ File.basename(adapter, '.rb') }"
+end

data/lib/wide_receiver/adapters/null_adapter.rb

@@ -0,0 +1,12 @@
+module WideReceiver
+  module Adapters
+    class NullAdapter
+      def initialize(channel, workers)
+      end
+
+      def work
+        sleep 0.001 # pause so the test can count threads
+      end
+    end
+  end
+end

data/lib/wide_receiver/adapters/redis_adapter.rb

@@ -0,0 +1,68 @@
+begin
+  require 'redis'
+  require 'redis-namespace'
+rescue LoadError => e
+end
+
+module WideReceiver
+  module Adapters
+    class RedisAdapter
+      attr_reader :config, :input, :error
+
+      def initialize(channel, workers, config: Config.instance)
+        @pattern = channel
+        @workers = workers.map { |w| Object.const_get(w) }
+        @config  = config
+
+        @input   = redis_connection
+        @error   = Redis::Namespace.new(:wide_receiver, redis: redis_connection)
+      end
+
+      def work
+        @input.psubscribe(@pattern) do |on|
+          on.pmessage do |pattern, channel, message|
+            send_workers channel, processed(message)
+          end
+        end
+      end
+
+      private
+
+      def processed(message)
+        case config.message_format
+        when :json
+          MultiJson.load(message)
+        else
+          message
+        end
+      end
+
+      def send_workers(channel, message)
+        @workers.each do |worker_class|
+          begin
+            worker_class.new.perform(channel, message)
+          rescue => e
+            @error.lpush 'failures', MultiJson.dump(
+              worker:     worker_class.to_s,
+              channel:    channel,
+              message:    message,
+              exception:  e.message
+            )
+          end
+        end
+      end
+
+      def redis_connection
+        Redis.new(redis_config(config.queue_uri))
+      end
+
+      def redis_config(uri)
+        {
+          host: uri.host,
+          port: uri.port,
+          db:   uri.path.to_s.scan(/\d+/).flatten.first
+        }.reject { |k,v| v.nil? }
+      end
+    end
+  end
+end

data/lib/wide_receiver/config.rb

@@ -0,0 +1,31 @@
+require_relative 'adapters'
+
+module WideReceiver
+  class Config
+    class NullUri < Struct.new(:scheme, :host, :port, :path)
+    end
+
+    attr_accessor :queue_url
+    attr_writer :message_format
+
+    def self.instance
+      @instance ||= new
+    end
+
+    def self.reset!
+      @instance = nil
+    end
+
+    def message_format
+      @message_format || :raw
+    end
+
+    def adapter
+      WideReceiver::Adapters.const_get(queue_uri.scheme.capitalize + 'Adapter')
+    end
+
+    def queue_uri
+      @uri ||= URI.parse(queue_url) rescue NullUri.new('null', nil, nil, nil)
+    end
+  end
+end

data/lib/wide_receiver/master.rb

@@ -0,0 +1,35 @@
+require 'thread'
+require_relative 'registry'
+require_relative 'config'
+
+Thread.abort_on_exception = true
+
+module WideReceiver
+  class Master
+    attr_reader :registry, :adapter
+
+    def initialize(registry: Registry.instance, adapter: Config.instance.adapter)
+      @adapter  = adapter
+      @registry = registry
+      @threads  = []
+    end
+
+    def start
+      registry.channels.each do |channel|
+        workers = registry[channel]
+
+        @threads << Thread.new do
+          adapter.new(channel, workers).work
+        end
+      end
+    end
+
+    def stop
+      @threads.each { |t| Thread.kill(t) }
+    end
+
+    def thread_count
+      @threads.size
+    end
+  end
+end

data/lib/wide_receiver/registry.rb

@@ -0,0 +1,30 @@
+require 'thread'
+
+module WideReceiver
+  class Registry
+    def self.instance
+      @instance ||= new
+    end
+
+    def initialize
+      @registry = Hash.new { |h,k| h[k] = [] }
+      @lock = Mutex.new
+    end
+
+    def register(worker_class, channels)
+      @lock.synchronize do
+        channels.each do |channel|
+          @registry[channel] << worker_class
+        end
+      end
+    end
+
+    def [](channel)
+      @lock.synchronize { @registry[channel] }
+    end
+
+    def channels
+      @lock.synchronize { @registry.keys }
+    end
+  end
+end

data/lib/wide_receiver/version.rb

@@ -0,0 +1,3 @@
+module WideReceiver
+  VERSION = "0.0.1"
+end

data/lib/wide_receiver/worker.rb

@@ -0,0 +1,14 @@
+require_relative 'registry'
+
+module WideReceiver
+  module Worker
+    def listen(*channels, registry: Registry.instance)
+      @channels = channels
+      registry.register(self.to_s, channels)
+    end
+
+    def channels
+      @channels
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+require 'pry'
+
+RSpec.configure do |config|
+  config.order = "random"
+
+  config.after :each do
+    if WideReceiver::Config.instance.adapter.to_s =~ /RedisAdapter/
+      redis = WideReceiver::Config.instance.adapter.new(:channel, []).error
+      redis.flushdb
+    end
+
+    WideReceiver::Config.reset!
+  end
+end

data/spec/wide_receiver/adapters/redis_adapter_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'wide_receiver/adapters/redis_adapter'
+
+class WideReceiver::Adapters::RedisAdapter
+  public :send_workers, :processed
+end
+
+class BrokenWorker
+  def perform(*args)
+    raise "kaboom"
+  end
+end
+
+describe WideReceiver::Adapters::RedisAdapter do
+
+  let(:worker_instance) { double }
+  let(:worker_class)    { double('WorkerClass', new: worker_instance) }
+  let(:config)          { double('Config', queue_uri: URI.parse('redis://localhost:6379/8')) }
+
+  it 'configures redis connection' do
+    adapter = described_class.new(:foo, [], config: config)
+    redis_client = adapter.input.client
+    expect(redis_client.host).to eq 'localhost'
+  end
+
+  it 'sends perform message to worker instances' do
+    Object.stub(:const_get).with('SomeClass').and_return(worker_class)
+    adapter = described_class.new(:foo, ['SomeClass'])
+
+    expect(worker_instance).to receive(:perform).with(19, 'breaker, breaker')
+    adapter.send_workers(19, 'breaker, breaker')
+  end
+
+  it 'parses JSON message if message_format is configured' do
+    adapter = described_class.new(:foo, [])
+    expect(adapter.processed('{"hello":"json"}')).to eq('{"hello":"json"}')
+  end
+
+  it 'parses JSON message if message_format is configured' do
+    config.stub(:message_format).and_return(:json)
+    adapter = described_class.new(:foo, [], config: config)
+    expect(adapter.processed('{"hello":"json"}')).to eq('hello' => 'json')
+  end
+
+  it 'records failed jobs in a queue' do
+    WideReceiver::Config.instance.queue_url = 'redis://localhost:6379/8'
+    adapter = described_class.new(:foo, ['BrokenWorker'])
+    expect {
+      adapter.send_workers('high-priority', 'hello world')
+    }.to_not raise_error
+    expect(adapter.error.llen 'failures').to eq 1
+  end
+
+end

data/spec/wide_receiver/config_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+require 'wide_receiver/config'
+
+describe WideReceiver::Config do
+
+  it 'returns adapter class for connection' do
+    config = described_class.new
+    config.queue_url = "redis://localhost:6379/5"
+
+    expect(config.adapter.to_s).to eq 'WideReceiver::Adapters::RedisAdapter'
+  end
+
+  it 'returns null object when queue url not specified' do
+    config = described_class.new
+    expect(config.queue_uri.scheme).to eq 'null'
+    expect(config.queue_uri.host).to be_nil
+  end
+
+end

data/spec/wide_receiver/master_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+require 'wide_receiver/master'
+
+describe WideReceiver::Master do
+  let(:registry) { double('Registry', channels: %w( 1 2 3 ), :'[]' => %w( NullWorker )) }
+  let(:subject)  { described_class.new(registry: registry, adapter: WideReceiver::Adapters::NullAdapter) }
+
+  it 'starts listener threads for registered channels' do
+    subject.start
+    expect(subject.thread_count).to eq 3
+  end
+
+end

data/spec/wide_receiver/registry_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+require 'wide_receiver/registry'
+require 'wide_receiver/worker'
+
+class ActivityWorker
+  extend WideReceiver::Worker
+  listen "*.reported.activity"
+end
+
+describe WideReceiver::Registry do
+
+  it 'connects workers with channels' do
+    registry = described_class.instance
+    expect(registry['*.reported.activity']).to eq %w( ActivityWorker )
+  end
+
+end

data/spec/wide_receiver/worker_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+require 'wide_receiver/worker'
+
+class TestWorker
+  extend WideReceiver::Worker
+  listen "*"
+end
+
+describe WideReceiver::Worker do
+
+  it 'listens to named channels' do
+    expect(TestWorker.channels).to eq ['*']
+  end
+
+end

data/wide_receiver.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'wide_receiver/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "wide_receiver"
+  spec.version       = WideReceiver::VERSION
+  spec.authors       = ["Solomon White"]
+  spec.email         = ["rubysolo@gmail.com"]
+  spec.summary       = %q{WideReceiver}
+  spec.description   = %q{Message bus fanout workers}
+  spec.homepage      = "https://github.com/rubysolo/wide_receiver"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "redis"
+  spec.add_dependency "redis-namespace"
+  spec.add_dependency "multi_json"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry-nav"
+end

metadata

@@ -0,0 +1,170 @@
+--- !ruby/object:Gem::Specification
+name: wide_receiver
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Solomon White
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Message bus fanout workers
+email:
+- rubysolo@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/wide_receiver.rb
+- lib/wide_receiver/adapters.rb
+- lib/wide_receiver/adapters/null_adapter.rb
+- lib/wide_receiver/adapters/redis_adapter.rb
+- lib/wide_receiver/config.rb
+- lib/wide_receiver/master.rb
+- lib/wide_receiver/registry.rb
+- lib/wide_receiver/version.rb
+- lib/wide_receiver/worker.rb
+- spec/spec_helper.rb
+- spec/wide_receiver/adapters/redis_adapter_spec.rb
+- spec/wide_receiver/config_spec.rb
+- spec/wide_receiver/master_spec.rb
+- spec/wide_receiver/registry_spec.rb
+- spec/wide_receiver/worker_spec.rb
+- wide_receiver.gemspec
+homepage: https://github.com/rubysolo/wide_receiver
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: WideReceiver
+test_files:
+- spec/spec_helper.rb
+- spec/wide_receiver/adapters/redis_adapter_spec.rb
+- spec/wide_receiver/config_spec.rb
+- spec/wide_receiver/master_spec.rb
+- spec/wide_receiver/registry_spec.rb
+- spec/wide_receiver/worker_spec.rb