wide_receiver 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 9f973638dac7294b4f2bfc94c2fc3c0d372a27c7
4
+ data.tar.gz: f57c4dafbc355f6a41bb6bedf0b861c33e1a2a4a
5
+ SHA512:
6
+ metadata.gz: d726e625d0fa7564265191f547c361d4ca5eba73197d7322640ea2a110bb6114a5291b61853833b1b78628b6c8d310b86091e9b37e6ab283c2b2f0fddb9df237
7
+ data.tar.gz: 3a2761f89de4cb934936e8b7ef5ac443fbdfd96c36e4f5ab251f3ec12cf66d05ad1c9d607a0f415eeb3f8e5b5173322611597a01c9ec23b855cfd205f3780d30
data/.gitignore ADDED
@@ -0,0 +1,19 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ bin
10
+ coverage
11
+ doc/
12
+ lib/bundler/man
13
+ pkg
14
+ rdoc
15
+ spec/reports
16
+ test/tmp
17
+ test/version_tmp
18
+ tmp
19
+ vendor/bundle
data/.travis.yml ADDED
@@ -0,0 +1,8 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.0.0
4
+ - 2.1.0
5
+ - 2.1.1
6
+ - rbx-2.1.1
7
+ services:
8
+ - redis-server
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in wide_receiver.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Solomon White
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,64 @@
1
+ # Wide Receiver
2
+
3
+ Wide Receiver is a work queue system in which the worker classes decide which
4
+ messages to process.
5
+
6
+ ## Installation
7
+
8
+ Add this line to your application's Gemfile:
9
+
10
+ gem 'wide_receiver'
11
+
12
+ And then execute:
13
+
14
+ $ bundle
15
+
16
+ Or install it yourself as:
17
+
18
+ $ gem install wide_receiver
19
+
20
+ ## Usage
21
+
22
+ Configure Wide Receiver to connect to your queuing system (currently only
23
+ supports Redis pubsub):
24
+
25
+ ```ruby
26
+ WideReceiver::Config.instance.queue_url = 'redis://localhost/6379/5'
27
+ ```
28
+
29
+ Create a worker class that extends `WideReceiver::Worker` and specifies which channel
30
+ patterns it wants to process:
31
+
32
+ ```ruby
33
+ class Rice
34
+ extend WideReceiver::Worker
35
+ listen 'montana.pass.*'
36
+
37
+ def perform(channel, message)
38
+ # do something with message
39
+ end
40
+ end
41
+ ```
42
+
43
+ Start the master process:
44
+
45
+ ```ruby
46
+ WideReceiver::Master.new.start
47
+ ```
48
+
49
+ This will start one thread per unique channel pattern. When a message is
50
+ received on a matching channel, each worker class that expressed interest in
51
+ that pattern will be instantiated, and the instance will receive the message.
52
+
53
+ ## TODO
54
+
55
+ - add support for RabbitMQ
56
+ - add proper command line runner
57
+
58
+ ## Contributing
59
+
60
+ 1. Fork it ( http://github.com/rubysolo/wide_receiver/fork )
61
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
62
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
63
+ 4. Push to the branch (`git push origin my-new-feature`)
64
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,13 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rspec/core/rake_task'
3
+
4
+ desc "Run specs"
5
+ task :spec do
6
+ RSpec::Core::RakeTask.new(:spec) do |t|
7
+ t.rspec_opts = %w{--colour --format progress}
8
+ t.pattern = 'spec/*_spec.rb'
9
+ end
10
+ end
11
+
12
+ desc "Default: run specs."
13
+ task default: :spec
@@ -0,0 +1,5 @@
1
+ require "wide_receiver/config"
2
+ require "wide_receiver/registry"
3
+ require "wide_receiver/worker"
4
+ require "wide_receiver/master"
5
+ require "wide_receiver/version"
@@ -0,0 +1,10 @@
1
+ require 'multi_json'
2
+
3
+ module WideReceiver
4
+ module Adapters
5
+ end
6
+ end
7
+
8
+ Dir[File.dirname(__FILE__) + '/adapters/*.rb'].each do |adapter|
9
+ require_relative "adapters/#{ File.basename(adapter, '.rb') }"
10
+ end
@@ -0,0 +1,12 @@
1
+ module WideReceiver
2
+ module Adapters
3
+ class NullAdapter
4
+ def initialize(channel, workers)
5
+ end
6
+
7
+ def work
8
+ sleep 0.001 # pause so the test can count threads
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,68 @@
1
+ begin
2
+ require 'redis'
3
+ require 'redis-namespace'
4
+ rescue LoadError => e
5
+ end
6
+
7
+ module WideReceiver
8
+ module Adapters
9
+ class RedisAdapter
10
+ attr_reader :config, :input, :error
11
+
12
+ def initialize(channel, workers, config: Config.instance)
13
+ @pattern = channel
14
+ @workers = workers.map { |w| Object.const_get(w) }
15
+ @config = config
16
+
17
+ @input = redis_connection
18
+ @error = Redis::Namespace.new(:wide_receiver, redis: redis_connection)
19
+ end
20
+
21
+ def work
22
+ @input.psubscribe(@pattern) do |on|
23
+ on.pmessage do |pattern, channel, message|
24
+ send_workers channel, processed(message)
25
+ end
26
+ end
27
+ end
28
+
29
+ private
30
+
31
+ def processed(message)
32
+ case config.message_format
33
+ when :json
34
+ MultiJson.load(message)
35
+ else
36
+ message
37
+ end
38
+ end
39
+
40
+ def send_workers(channel, message)
41
+ @workers.each do |worker_class|
42
+ begin
43
+ worker_class.new.perform(channel, message)
44
+ rescue => e
45
+ @error.lpush 'failures', MultiJson.dump(
46
+ worker: worker_class.to_s,
47
+ channel: channel,
48
+ message: message,
49
+ exception: e.message
50
+ )
51
+ end
52
+ end
53
+ end
54
+
55
+ def redis_connection
56
+ Redis.new(redis_config(config.queue_uri))
57
+ end
58
+
59
+ def redis_config(uri)
60
+ {
61
+ host: uri.host,
62
+ port: uri.port,
63
+ db: uri.path.to_s.scan(/\d+/).flatten.first
64
+ }.reject { |k,v| v.nil? }
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,31 @@
1
+ require_relative 'adapters'
2
+
3
+ module WideReceiver
4
+ class Config
5
+ class NullUri < Struct.new(:scheme, :host, :port, :path)
6
+ end
7
+
8
+ attr_accessor :queue_url
9
+ attr_writer :message_format
10
+
11
+ def self.instance
12
+ @instance ||= new
13
+ end
14
+
15
+ def self.reset!
16
+ @instance = nil
17
+ end
18
+
19
+ def message_format
20
+ @message_format || :raw
21
+ end
22
+
23
+ def adapter
24
+ WideReceiver::Adapters.const_get(queue_uri.scheme.capitalize + 'Adapter')
25
+ end
26
+
27
+ def queue_uri
28
+ @uri ||= URI.parse(queue_url) rescue NullUri.new('null', nil, nil, nil)
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,35 @@
1
+ require 'thread'
2
+ require_relative 'registry'
3
+ require_relative 'config'
4
+
5
+ Thread.abort_on_exception = true
6
+
7
+ module WideReceiver
8
+ class Master
9
+ attr_reader :registry, :adapter
10
+
11
+ def initialize(registry: Registry.instance, adapter: Config.instance.adapter)
12
+ @adapter = adapter
13
+ @registry = registry
14
+ @threads = []
15
+ end
16
+
17
+ def start
18
+ registry.channels.each do |channel|
19
+ workers = registry[channel]
20
+
21
+ @threads << Thread.new do
22
+ adapter.new(channel, workers).work
23
+ end
24
+ end
25
+ end
26
+
27
+ def stop
28
+ @threads.each { |t| Thread.kill(t) }
29
+ end
30
+
31
+ def thread_count
32
+ @threads.size
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,30 @@
1
+ require 'thread'
2
+
3
+ module WideReceiver
4
+ class Registry
5
+ def self.instance
6
+ @instance ||= new
7
+ end
8
+
9
+ def initialize
10
+ @registry = Hash.new { |h,k| h[k] = [] }
11
+ @lock = Mutex.new
12
+ end
13
+
14
+ def register(worker_class, channels)
15
+ @lock.synchronize do
16
+ channels.each do |channel|
17
+ @registry[channel] << worker_class
18
+ end
19
+ end
20
+ end
21
+
22
+ def [](channel)
23
+ @lock.synchronize { @registry[channel] }
24
+ end
25
+
26
+ def channels
27
+ @lock.synchronize { @registry.keys }
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,3 @@
1
+ module WideReceiver
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,14 @@
1
+ require_relative 'registry'
2
+
3
+ module WideReceiver
4
+ module Worker
5
+ def listen(*channels, registry: Registry.instance)
6
+ @channels = channels
7
+ registry.register(self.to_s, channels)
8
+ end
9
+
10
+ def channels
11
+ @channels
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ require 'pry'
2
+
3
+ RSpec.configure do |config|
4
+ config.order = "random"
5
+
6
+ config.after :each do
7
+ if WideReceiver::Config.instance.adapter.to_s =~ /RedisAdapter/
8
+ redis = WideReceiver::Config.instance.adapter.new(:channel, []).error
9
+ redis.flushdb
10
+ end
11
+
12
+ WideReceiver::Config.reset!
13
+ end
14
+ end
@@ -0,0 +1,54 @@
1
+ require 'spec_helper'
2
+ require 'wide_receiver/adapters/redis_adapter'
3
+
4
+ class WideReceiver::Adapters::RedisAdapter
5
+ public :send_workers, :processed
6
+ end
7
+
8
+ class BrokenWorker
9
+ def perform(*args)
10
+ raise "kaboom"
11
+ end
12
+ end
13
+
14
+ describe WideReceiver::Adapters::RedisAdapter do
15
+
16
+ let(:worker_instance) { double }
17
+ let(:worker_class) { double('WorkerClass', new: worker_instance) }
18
+ let(:config) { double('Config', queue_uri: URI.parse('redis://localhost:6379/8')) }
19
+
20
+ it 'configures redis connection' do
21
+ adapter = described_class.new(:foo, [], config: config)
22
+ redis_client = adapter.input.client
23
+ expect(redis_client.host).to eq 'localhost'
24
+ end
25
+
26
+ it 'sends perform message to worker instances' do
27
+ Object.stub(:const_get).with('SomeClass').and_return(worker_class)
28
+ adapter = described_class.new(:foo, ['SomeClass'])
29
+
30
+ expect(worker_instance).to receive(:perform).with(19, 'breaker, breaker')
31
+ adapter.send_workers(19, 'breaker, breaker')
32
+ end
33
+
34
+ it 'parses JSON message if message_format is configured' do
35
+ adapter = described_class.new(:foo, [])
36
+ expect(adapter.processed('{"hello":"json"}')).to eq('{"hello":"json"}')
37
+ end
38
+
39
+ it 'parses JSON message if message_format is configured' do
40
+ config.stub(:message_format).and_return(:json)
41
+ adapter = described_class.new(:foo, [], config: config)
42
+ expect(adapter.processed('{"hello":"json"}')).to eq('hello' => 'json')
43
+ end
44
+
45
+ it 'records failed jobs in a queue' do
46
+ WideReceiver::Config.instance.queue_url = 'redis://localhost:6379/8'
47
+ adapter = described_class.new(:foo, ['BrokenWorker'])
48
+ expect {
49
+ adapter.send_workers('high-priority', 'hello world')
50
+ }.to_not raise_error
51
+ expect(adapter.error.llen 'failures').to eq 1
52
+ end
53
+
54
+ end
@@ -0,0 +1,19 @@
1
+ require 'spec_helper'
2
+ require 'wide_receiver/config'
3
+
4
+ describe WideReceiver::Config do
5
+
6
+ it 'returns adapter class for connection' do
7
+ config = described_class.new
8
+ config.queue_url = "redis://localhost:6379/5"
9
+
10
+ expect(config.adapter.to_s).to eq 'WideReceiver::Adapters::RedisAdapter'
11
+ end
12
+
13
+ it 'returns null object when queue url not specified' do
14
+ config = described_class.new
15
+ expect(config.queue_uri.scheme).to eq 'null'
16
+ expect(config.queue_uri.host).to be_nil
17
+ end
18
+
19
+ end
@@ -0,0 +1,13 @@
1
+ require 'spec_helper'
2
+ require 'wide_receiver/master'
3
+
4
+ describe WideReceiver::Master do
5
+ let(:registry) { double('Registry', channels: %w( 1 2 3 ), :'[]' => %w( NullWorker )) }
6
+ let(:subject) { described_class.new(registry: registry, adapter: WideReceiver::Adapters::NullAdapter) }
7
+
8
+ it 'starts listener threads for registered channels' do
9
+ subject.start
10
+ expect(subject.thread_count).to eq 3
11
+ end
12
+
13
+ end
@@ -0,0 +1,17 @@
1
+ require 'spec_helper'
2
+ require 'wide_receiver/registry'
3
+ require 'wide_receiver/worker'
4
+
5
+ class ActivityWorker
6
+ extend WideReceiver::Worker
7
+ listen "*.reported.activity"
8
+ end
9
+
10
+ describe WideReceiver::Registry do
11
+
12
+ it 'connects workers with channels' do
13
+ registry = described_class.instance
14
+ expect(registry['*.reported.activity']).to eq %w( ActivityWorker )
15
+ end
16
+
17
+ end
@@ -0,0 +1,15 @@
1
+ require 'spec_helper'
2
+ require 'wide_receiver/worker'
3
+
4
+ class TestWorker
5
+ extend WideReceiver::Worker
6
+ listen "*"
7
+ end
8
+
9
+ describe WideReceiver::Worker do
10
+
11
+ it 'listens to named channels' do
12
+ expect(TestWorker.channels).to eq ['*']
13
+ end
14
+
15
+ end
@@ -0,0 +1,29 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'wide_receiver/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "wide_receiver"
8
+ spec.version = WideReceiver::VERSION
9
+ spec.authors = ["Solomon White"]
10
+ spec.email = ["rubysolo@gmail.com"]
11
+ spec.summary = %q{WideReceiver}
12
+ spec.description = %q{Message bus fanout workers}
13
+ spec.homepage = "https://github.com/rubysolo/wide_receiver"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files`.split($/)
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_dependency "redis"
22
+ spec.add_dependency "redis-namespace"
23
+ spec.add_dependency "multi_json"
24
+
25
+ spec.add_development_dependency "bundler", "~> 1.5"
26
+ spec.add_development_dependency "rake"
27
+ spec.add_development_dependency "rspec"
28
+ spec.add_development_dependency "pry-nav"
29
+ end
metadata ADDED
@@ -0,0 +1,170 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: wide_receiver
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Solomon White
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-03-14 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: redis
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: redis-namespace
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: multi_json
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.5'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.5'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rake
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rspec
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: pry-nav
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ description: Message bus fanout workers
112
+ email:
113
+ - rubysolo@gmail.com
114
+ executables: []
115
+ extensions: []
116
+ extra_rdoc_files: []
117
+ files:
118
+ - ".gitignore"
119
+ - ".travis.yml"
120
+ - Gemfile
121
+ - LICENSE.txt
122
+ - README.md
123
+ - Rakefile
124
+ - lib/wide_receiver.rb
125
+ - lib/wide_receiver/adapters.rb
126
+ - lib/wide_receiver/adapters/null_adapter.rb
127
+ - lib/wide_receiver/adapters/redis_adapter.rb
128
+ - lib/wide_receiver/config.rb
129
+ - lib/wide_receiver/master.rb
130
+ - lib/wide_receiver/registry.rb
131
+ - lib/wide_receiver/version.rb
132
+ - lib/wide_receiver/worker.rb
133
+ - spec/spec_helper.rb
134
+ - spec/wide_receiver/adapters/redis_adapter_spec.rb
135
+ - spec/wide_receiver/config_spec.rb
136
+ - spec/wide_receiver/master_spec.rb
137
+ - spec/wide_receiver/registry_spec.rb
138
+ - spec/wide_receiver/worker_spec.rb
139
+ - wide_receiver.gemspec
140
+ homepage: https://github.com/rubysolo/wide_receiver
141
+ licenses:
142
+ - MIT
143
+ metadata: {}
144
+ post_install_message:
145
+ rdoc_options: []
146
+ require_paths:
147
+ - lib
148
+ required_ruby_version: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ required_rubygems_version: !ruby/object:Gem::Requirement
154
+ requirements:
155
+ - - ">="
156
+ - !ruby/object:Gem::Version
157
+ version: '0'
158
+ requirements: []
159
+ rubyforge_project:
160
+ rubygems_version: 2.2.0
161
+ signing_key:
162
+ specification_version: 4
163
+ summary: WideReceiver
164
+ test_files:
165
+ - spec/spec_helper.rb
166
+ - spec/wide_receiver/adapters/redis_adapter_spec.rb
167
+ - spec/wide_receiver/config_spec.rb
168
+ - spec/wide_receiver/master_spec.rb
169
+ - spec/wide_receiver/registry_spec.rb
170
+ - spec/wide_receiver/worker_spec.rb