firehose 0.0.16 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.rspec +1 -1
- data/README.md +6 -7
- data/firehose.gemspec +1 -1
- data/lib/firehose.rb +1 -16
- data/lib/firehose/cli.rb +4 -16
- data/lib/firehose/publisher.rb +8 -8
- data/lib/firehose/rack.rb +32 -31
- data/lib/firehose/subscription.rb +56 -46
- data/lib/firehose/version.rb +2 -2
- data/spec/integrations/thin_spec.rb +6 -6
- data/spec/lib/broker_spec.rb +23 -23
- data/spec/lib/consumer_spec.rb +56 -56
- data/spec/spec_helper.rb +1 -4
- metadata +30 -34
- data/lib/firehose/broker.rb +0 -34
- data/lib/firehose/consumer.rb +0 -49
- data/spec/integrations/amqp_resources_spec.rb +0 -51
data/.rspec
CHANGED
@@ -1 +1 @@
|
|
1
|
-
--colour
|
1
|
+
--colour --backtrace
|
data/README.md
CHANGED
@@ -13,11 +13,11 @@ Firehose is both a Rack application and JavasScript library that makes building
|
|
13
13
|
|
14
14
|
# Getting Started
|
15
15
|
|
16
|
-
First, you'll need to install and run
|
16
|
+
First, you'll need to install and run Redis.
|
17
17
|
|
18
18
|
```sh
|
19
|
-
$ apt-get install
|
20
|
-
$ brew install
|
19
|
+
$ apt-get install redis # Install on Ubuntu
|
20
|
+
$ brew install redis # Install on Mac Homebrew
|
21
21
|
```
|
22
22
|
|
23
23
|
Then install the gem.
|
@@ -34,7 +34,7 @@ Now fire up the server.
|
|
34
34
|
$ firehose server
|
35
35
|
>> Thin web server (v1.3.1 codename Triple Espresso)
|
36
36
|
>> Maximum connections set to 1024
|
37
|
-
>> Listening on 127.0.0.1:
|
37
|
+
>> Listening on 127.0.0.1:7474, CTRL+C to stop
|
38
38
|
```
|
39
39
|
|
40
40
|
In case you're wondering, the Firehose application server runs the Rack app `Firehose::Rack::App.new` inside of Thin.
|
@@ -87,15 +87,14 @@ new Firehose.Consumer({
|
|
87
87
|
},
|
88
88
|
// Note that we do NOT specify a protocol here because we don't
|
89
89
|
// know that yet.
|
90
|
-
uri: '//localhost:
|
90
|
+
uri: '//localhost:7474/hello'
|
91
91
|
}).connect();
|
92
92
|
```
|
93
93
|
|
94
94
|
Then publish another message.
|
95
95
|
|
96
|
-
|
97
96
|
```sh
|
98
|
-
$ curl -X PUT -d "This is almost magical" "http://localhost:
|
97
|
+
$ curl -X PUT -d "\"This is almost magical\"" "http://localhost:7474/hello"
|
99
98
|
```
|
100
99
|
|
101
100
|
# How is it different from socket.io?
|
data/firehose.gemspec
CHANGED
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
|
|
20
20
|
|
21
21
|
# specify any dependencies here; for example:
|
22
22
|
s.add_runtime_dependency "eventmachine", ">= 1.0.0.beta"
|
23
|
-
s.add_runtime_dependency "
|
23
|
+
s.add_runtime_dependency "em-hiredis"
|
24
24
|
s.add_runtime_dependency "thin"
|
25
25
|
s.add_runtime_dependency "thor"
|
26
26
|
s.add_runtime_dependency "faraday"
|
data/lib/firehose.rb
CHANGED
@@ -1,31 +1,16 @@
|
|
1
1
|
require 'firehose/version'
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'em-hiredis'
|
4
4
|
require 'logger'
|
5
5
|
|
6
6
|
module Firehose
|
7
7
|
autoload :Subscription, 'firehose/subscription'
|
8
8
|
autoload :Publisher, 'firehose/publisher'
|
9
9
|
autoload :Producer, 'firehose/producer'
|
10
|
-
autoload :Consumer, 'firehose/consumer'
|
11
10
|
autoload :Default, 'firehose/default'
|
12
|
-
autoload :Broker, 'firehose/broker'
|
13
11
|
autoload :Rack, 'firehose/rack'
|
14
12
|
autoload :CLI, 'firehose/cli'
|
15
13
|
|
16
|
-
# TODO move this into a configuration or session class.
|
17
|
-
# Hang on to AMQP configuration settings.
|
18
|
-
def self.amqp
|
19
|
-
@amqp ||= Struct.new(:connection).new(AMQP.connect)
|
20
|
-
end
|
21
|
-
|
22
|
-
# TODO figure out a better way to memoize AMQP connection for production runtimes, and
|
23
|
-
# make it resetable for testing environment. Some sort of Firehose::Session object is probably
|
24
|
-
# in order
|
25
|
-
def self.reset!
|
26
|
-
@amqp = nil
|
27
|
-
end
|
28
|
-
|
29
14
|
# Logging
|
30
15
|
def self.logger
|
31
16
|
@logger ||= Logger.new($stdout)
|
data/lib/firehose/cli.rb
CHANGED
@@ -8,27 +8,15 @@ module Firehose
|
|
8
8
|
method_option :host, :type => :string, :default => '0.0.0.0', :required => true, :aliases => '-h'
|
9
9
|
|
10
10
|
def server
|
11
|
-
broker = Firehose::Broker.new
|
12
|
-
|
13
11
|
server = Thin::Server.new(options[:host], options[:port]) do
|
14
|
-
|
15
|
-
# connection reference issues. I'd like to have this ancillary stuff accessiable via
|
16
|
-
# a different port or even a socket.
|
17
|
-
map '/_firehose/stats' do
|
18
|
-
run Proc.new {
|
19
|
-
[200, {'Content-Type' => 'text/plain'}, [broker.stats.inspect]]
|
20
|
-
}
|
21
|
-
end
|
22
|
-
|
23
|
-
map '/' do
|
24
|
-
run Firehose::Rack::App.new(broker)
|
25
|
-
end
|
12
|
+
run Firehose::Rack::App.new
|
26
13
|
end
|
27
14
|
|
28
15
|
begin
|
29
16
|
server.start!
|
30
|
-
rescue
|
31
|
-
Firehose.logger.error "Unable to connect to
|
17
|
+
rescue RuntimeError
|
18
|
+
Firehose.logger.error "Unable to connect to Redis, are you sure it's running?"
|
19
|
+
raise
|
32
20
|
end
|
33
21
|
end
|
34
22
|
end
|
data/lib/firehose/publisher.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
module Firehose
|
2
2
|
class Publisher
|
3
|
-
def publish(
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
3
|
+
def publish(channel, message)
|
4
|
+
Firehose.logger.debug "Redis publishing `#{message}` to `#{channel}`"
|
5
|
+
redis.publish(channel, message).errback { raise 'Error publishing' }
|
6
|
+
end
|
7
|
+
|
8
|
+
private
|
9
|
+
def redis
|
10
|
+
@redis ||= EM::Hiredis.connect
|
11
11
|
end
|
12
12
|
end
|
13
13
|
end
|
data/lib/firehose/rack.rb
CHANGED
@@ -5,16 +5,14 @@ module Firehose
|
|
5
5
|
AsyncResponse = [-1, {}, []]
|
6
6
|
|
7
7
|
class HttpLongPoll
|
8
|
-
def initialize(broker)
|
9
|
-
@broker = broker
|
10
|
-
end
|
11
|
-
|
12
8
|
def call(env)
|
13
9
|
req = ::Rack::Request.new(env)
|
14
10
|
cid = req.params['cid']
|
15
11
|
path = req.path
|
16
12
|
method = req.request_method
|
17
13
|
timeout = 30
|
14
|
+
queue_name = "#{cid}@#{path}"
|
15
|
+
|
18
16
|
# TODO seperate out CORS logic as an async middleware with a Goliath web server.
|
19
17
|
cors_origin = env['HTTP_ORIGIN']
|
20
18
|
cors_headers = {
|
@@ -33,20 +31,23 @@ module Firehose
|
|
33
31
|
response_headers = cors_origin ? cors_headers : {}
|
34
32
|
|
35
33
|
# Setup a subscription with a client id. We haven't subscribed yet here.
|
36
|
-
|
34
|
+
if queue = queues[queue_name]
|
35
|
+
queue.live
|
36
|
+
else
|
37
|
+
queue = queues[queue_name] = Firehose::Subscription::Queue.new(cid, path)
|
38
|
+
end
|
37
39
|
|
38
40
|
# Setup a timeout timer to tell clients that time out that everything is OK
|
39
41
|
# and they should come back for more
|
40
|
-
|
42
|
+
long_poll_timer = EM::Timer.new(timeout) do
|
41
43
|
# We send a 204 OK to tell the client to reconnect.
|
42
44
|
env['async.callback'].call [204, response_headers, []]
|
43
45
|
Firehose.logger.debug "HTTP wait `#{cid}@#{path}` timed out"
|
44
46
|
end
|
45
47
|
|
46
48
|
# Ok, now subscribe to the subscription.
|
47
|
-
|
48
|
-
|
49
|
-
consumer.unsubscribe
|
49
|
+
queue.pop do |message, subscription|
|
50
|
+
long_poll_timer.cancel # Turn off the heart beat so we don't execute any of that business.
|
50
51
|
env['async.callback'].call [200, response_headers, [message]]
|
51
52
|
Firehose.logger.debug "HTTP sent `#{message}` to `#{cid}@#{path}`"
|
52
53
|
end
|
@@ -55,7 +56,11 @@ module Firehose
|
|
55
56
|
# Unsubscribe from the subscription if its still open and something bad happened
|
56
57
|
# or the heart beat triggered before we could finish.
|
57
58
|
env['async.close'].callback do
|
58
|
-
|
59
|
+
# Kill queue if we don't hear back in 30s
|
60
|
+
queue.kill timeout do
|
61
|
+
Firehose.logger.debug "Deleting queue to `#{queue_name}`"
|
62
|
+
queues.delete queue_name
|
63
|
+
end
|
59
64
|
Firehose.logger.debug "HTTP connection `#{cid}@#{path}` closing"
|
60
65
|
end
|
61
66
|
end
|
@@ -67,7 +72,7 @@ module Firehose
|
|
67
72
|
when 'PUT'
|
68
73
|
body = env['rack.input'].read
|
69
74
|
Firehose.logger.debug "HTTP published `#{body}` to `#{path}`"
|
70
|
-
|
75
|
+
publisher.publish(path, body)
|
71
76
|
|
72
77
|
[202, {}, []]
|
73
78
|
else
|
@@ -75,23 +80,28 @@ module Firehose
|
|
75
80
|
[501, {'Content-Type' => 'text/plain'}, ["#{method} not supported."]]
|
76
81
|
end
|
77
82
|
end
|
83
|
+
|
84
|
+
private
|
85
|
+
def publisher
|
86
|
+
@publisher ||= Firehose::Publisher.new
|
87
|
+
end
|
88
|
+
|
89
|
+
def queues
|
90
|
+
@queues ||= {}
|
91
|
+
end
|
78
92
|
end
|
79
93
|
|
80
94
|
class WebSocket < ::Rack::WebSocket::Application
|
81
|
-
attr_reader :cid, :path
|
95
|
+
attr_reader :cid, :path, :subscription
|
82
96
|
|
83
|
-
def initialize(broker)
|
84
|
-
@broker = broker
|
85
|
-
end
|
86
|
-
|
87
97
|
# Subscribe to a path and make some magic happen, mmkmay?
|
88
98
|
def on_open(env)
|
89
99
|
req = ::Rack::Request.new(env)
|
90
100
|
@cid = req.params['cid']
|
91
101
|
@path = req.path
|
102
|
+
@subscription = Firehose::Subscription.new(cid, path)
|
92
103
|
|
93
|
-
|
94
|
-
@consumer.subscribe_to path do |message|
|
104
|
+
subscription.subscribe do |message, subscription|
|
95
105
|
Firehose.logger.debug "WS sent `#{message}` to `#{cid}@#{path}`"
|
96
106
|
send_data message
|
97
107
|
end
|
@@ -100,37 +110,28 @@ module Firehose
|
|
100
110
|
|
101
111
|
# Delete the subscription if the thing even happened.
|
102
112
|
def on_close(env)
|
103
|
-
|
113
|
+
subscription.unsubscribe
|
104
114
|
Firehose.logger.debug "WS connection `#{cid}@#{path}` closing"
|
105
115
|
end
|
106
116
|
|
107
117
|
# Log websocket level errors
|
108
118
|
def on_error(env, error)
|
109
|
-
Firehose.logger.error "WS connection `#{cid}@#{path}` error `#{error}`: #{
|
110
|
-
@consumer.unsubscribe if @consumer
|
119
|
+
Firehose.logger.error "WS connection `#{cid}@#{path}` error `#{error}`: #{error.backtrace}"
|
111
120
|
end
|
112
121
|
end
|
113
122
|
|
114
123
|
class App
|
115
|
-
# Firehose broker that will be used to pub/sub messages.
|
116
|
-
attr_reader :broker
|
117
|
-
|
118
|
-
# Fire up a default broker if one is not specified.
|
119
|
-
def initialize(broker = Firehose::Broker.new)
|
120
|
-
@broker = broker
|
121
|
-
end
|
122
|
-
|
123
124
|
def call(env)
|
124
125
|
websocket_request?(env) ? websocket.call(env) : http_long_poll.call(env)
|
125
126
|
end
|
126
127
|
|
127
128
|
private
|
128
129
|
def websocket
|
129
|
-
|
130
|
+
WebSocket.new
|
130
131
|
end
|
131
132
|
|
132
133
|
def http_long_poll
|
133
|
-
@http_long_poll ||= HttpLongPoll.new
|
134
|
+
@http_long_poll ||= HttpLongPoll.new
|
134
135
|
end
|
135
136
|
|
136
137
|
def websocket_request?(env)
|
@@ -1,65 +1,75 @@
|
|
1
1
|
module Firehose
|
2
2
|
class Subscription
|
3
|
-
#
|
4
|
-
|
5
|
-
# TODO should the Consumer handle TTL?
|
6
|
-
TTL = 15000
|
7
|
-
|
8
|
-
# Time to live for the amqp_queue on the server after the subscription is canceled. This
|
9
|
-
# is mostly for flakey connections where the client may reconnect after *ttl* and continue
|
10
|
-
# receiving messages.
|
11
|
-
attr_accessor :ttl
|
12
|
-
|
13
|
-
# Consumer and channel for the subscription.
|
14
|
-
attr_reader :consumer
|
3
|
+
# consumer_id and channel for the subscription.
|
4
|
+
attr_reader :consumer_id
|
15
5
|
|
16
6
|
# Channel that we'll use for the pub-sub activity. This probably maps to an URL
|
17
7
|
attr_reader :channel
|
18
8
|
|
19
|
-
def initialize(
|
20
|
-
@
|
9
|
+
def initialize(consumer_id, channel)
|
10
|
+
@consumer_id, @channel = consumer_id, channel
|
21
11
|
end
|
22
12
|
|
23
|
-
#
|
24
|
-
#
|
13
|
+
# Subscribe to messages on the backend to fill up the subscription queue. consumer_ids of the messages
|
14
|
+
# will queue up units of "work" to process data from the subscription.
|
25
15
|
def subscribe(&block)
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
# When we get a message, we want to remove the consumer from the amqp_queue so that the x-expires
|
33
|
-
# ttl starts ticking down. On the reconnect, the consumer connects to the amqp_queue and resets the
|
34
|
-
# timer on x-expires... in theory at least.
|
35
|
-
@amqp_consumer = AMQP::Consumer.new(amqp_channel, amqp_queue, consumer.guid)
|
36
|
-
@amqp_consumer.on_delivery do |metadata, message|
|
37
|
-
Firehose.logger.debug "AMQP delivering `#{message}` to `#{consumer.guid}@#{channel}`"
|
38
|
-
block.call(message, self)
|
39
|
-
# The ack needs to go after the block is called. This makes sure that all processing
|
40
|
-
# happens downstream before we remove it from the amqp_queue entirely.
|
41
|
-
metadata.ack
|
42
|
-
end.consume
|
43
|
-
Firehose.logger.debug "AMQP subscribed to `#{consumer.guid}@#{channel}`"
|
16
|
+
redis.subscribe(channel)
|
17
|
+
redis.on(:message) do |channel, message|
|
18
|
+
Firehose.logger.debug "Redis recieved `#{message}` to `#{consumer_id}@#{channel}`"
|
19
|
+
block.call message, self
|
20
|
+
end
|
21
|
+
Firehose.logger.debug "Redis subscribed to `#{consumer_id}@#{channel}`"
|
44
22
|
self # Return the subscription for chaining.
|
45
23
|
end
|
46
24
|
|
47
|
-
def unsubscribe
|
48
|
-
Firehose.logger.debug "
|
49
|
-
|
50
|
-
|
25
|
+
def unsubscribe(&block)
|
26
|
+
Firehose.logger.debug "Redis unsubscribed from `#{consumer_id}@#{channel}`"
|
27
|
+
redis.close
|
28
|
+
block.call(self) if block
|
29
|
+
self
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
def redis
|
34
|
+
@redis ||= EM::Hiredis.connect
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
# Queue subscription messages so that we can remember and/or operate on them
|
39
|
+
class Subscription::Queue
|
40
|
+
attr_reader :subscription, :channel
|
41
|
+
|
42
|
+
def initialize(consumer_id, channel)
|
43
|
+
@subscription = Subscription.new(consumer_id, channel)
|
44
|
+
# Start the subscription and start dropping mesasge onto the queue
|
45
|
+
subscription.subscribe do |message|
|
46
|
+
queue.push message
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
# Pop an item off the subscription queue so we can work on it.
|
51
|
+
def pop(&block)
|
52
|
+
queue.pop do |message|
|
53
|
+
block.call message, subscription
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Kill the queue in n seconds.
|
58
|
+
def kill(ttl=0, &block)
|
59
|
+
if ttl.zero?
|
60
|
+
subscription.unsubscribe &block
|
61
|
+
else
|
62
|
+
@timer = EM::Timer.new(ttl){ kill 0 }
|
63
|
+
end
|
51
64
|
end
|
52
65
|
|
53
|
-
|
54
|
-
|
55
|
-
def on_unsubscribe(&block)
|
56
|
-
@unsubscribe_callback = block
|
66
|
+
def live
|
67
|
+
@timer.cancel if @timer
|
57
68
|
end
|
58
69
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
@ttl ||= TTL
|
70
|
+
private
|
71
|
+
def queue
|
72
|
+
@queue ||= EM::Queue.new
|
63
73
|
end
|
64
74
|
end
|
65
75
|
end
|
data/lib/firehose/version.rb
CHANGED
@@ -12,12 +12,11 @@ describe Firehose::Rack do
|
|
12
12
|
end
|
13
13
|
|
14
14
|
let(:app) { Firehose::Rack::App.new }
|
15
|
-
let(:messages) { (1..
|
15
|
+
let(:messages) { (1..2000).map(&:to_s) }
|
16
16
|
let(:channel) { "/firehose/integration/#{Time.now.to_i}" }
|
17
17
|
let(:uri) { Firehose::Default::URI }
|
18
18
|
let(:http_url) { "http://#{uri.host}:#{uri.port}#{channel}" }
|
19
19
|
let(:ws_url) { "ws://#{uri.host}:#{uri.port}#{channel}" }
|
20
|
-
let(:cid) { "client-#{Time.now.to_i}" }
|
21
20
|
|
22
21
|
it "should pub-sub http and websockets" do
|
23
22
|
# Setup variables that we'll use after we turn off EM to validate our
|
@@ -39,7 +38,7 @@ describe Firehose::Rack do
|
|
39
38
|
|
40
39
|
# Lets have an HTTP Long poll client
|
41
40
|
http_long_poll = Proc.new do
|
42
|
-
http = EM::HttpRequest.new(http_url).get(:query => {'cid' =>
|
41
|
+
http = EM::HttpRequest.new(http_url).get(:query => {'cid' => 'alpha'})
|
43
42
|
http.errback { em.stop }
|
44
43
|
http.callback do
|
45
44
|
received_http << http.response
|
@@ -53,7 +52,7 @@ describe Firehose::Rack do
|
|
53
52
|
|
54
53
|
# And test a web socket client too, at the same time.
|
55
54
|
websocket = Proc.new do
|
56
|
-
ws = EventMachine::WebSocketClient.connect(ws_url)
|
55
|
+
ws = EventMachine::WebSocketClient.connect("#{ws_url}?cid=bravo")
|
57
56
|
ws.errback { em.stop }
|
58
57
|
ws.stream do |msg|
|
59
58
|
received_ws << msg
|
@@ -64,7 +63,8 @@ describe Firehose::Rack do
|
|
64
63
|
# Great, we have all the pieces in order, lets run this thing in the reactor.
|
65
64
|
em do
|
66
65
|
# Start the server
|
67
|
-
::Thin::Server.new('0.0.0.0', uri.port, app)
|
66
|
+
server = ::Thin::Server.new('0.0.0.0', uri.port, app)
|
67
|
+
server.start
|
68
68
|
|
69
69
|
# Start the http_long_pollr.
|
70
70
|
http_long_poll.call
|
@@ -75,7 +75,7 @@ describe Firehose::Rack do
|
|
75
75
|
end
|
76
76
|
|
77
77
|
# When EM stops, these assertions will be made.
|
78
|
-
received_http.should =~ messages
|
79
78
|
received_ws.should =~ messages
|
79
|
+
received_http.should =~ messages
|
80
80
|
end
|
81
81
|
end
|
data/spec/lib/broker_spec.rb
CHANGED
@@ -1,30 +1,30 @@
|
|
1
|
-
require 'spec_helper'
|
1
|
+
# require 'spec_helper'
|
2
2
|
|
3
|
-
describe Firehose::Broker do
|
4
|
-
|
3
|
+
# describe Firehose::Broker do
|
4
|
+
# include EM::TestHelper
|
5
5
|
|
6
|
-
|
6
|
+
# let(:broker) { Firehose::Broker.new }
|
7
7
|
|
8
|
-
|
9
|
-
|
8
|
+
# it "should unsubscibe consumers and remove them from the collection" do
|
9
|
+
# stats = nil
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
11
|
+
# em do
|
12
|
+
# broker.consumer('1').subscribe_to('/the-channel')
|
13
|
+
# broker.consumer('2').subscribe_to('/the-channel')
|
14
|
+
# broker.consumer('2').subscribe_to('/a-channel')
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
16
|
+
# em.add_timer(1) do
|
17
|
+
# stats = broker.stats
|
18
|
+
# broker.stop
|
19
|
+
# em.stop
|
20
|
+
# end
|
21
|
+
# end
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
# stats.should == {
|
24
|
+
# '1' => {'subscriptions' => ['/the-channel'] },
|
25
|
+
# '2' => {'subscriptions' => ['/the-channel', '/a-channel']}
|
26
|
+
# }
|
27
27
|
|
28
|
-
|
29
|
-
|
30
|
-
end
|
28
|
+
# broker.stats.should == {}
|
29
|
+
# end
|
30
|
+
# end
|
data/spec/lib/consumer_spec.rb
CHANGED
@@ -1,66 +1,66 @@
|
|
1
|
-
require 'spec_helper'
|
1
|
+
# require 'spec_helper'
|
2
2
|
|
3
|
-
describe Firehose::Consumer do
|
4
|
-
|
3
|
+
# describe Firehose::Consumer do
|
4
|
+
# include EM::TestHelper
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
# let(:consumer) { Firehose::Consumer.new }
|
7
|
+
# let(:publisher) { Firehose::Publisher.new }
|
8
|
+
# let(:channel) { '/papa-smurf' }
|
9
|
+
# let(:another_channel) { '/mama-smurf' }
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
11
|
+
# describe "subscriptions" do
|
12
|
+
# it "should subscribe to channel" do
|
13
|
+
# sent, recieved = 'hi', nil
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
15
|
+
# em do
|
16
|
+
# consumer.subscribe_to channel do |msg|
|
17
|
+
# recieved = msg
|
18
|
+
# em.stop
|
19
|
+
# end
|
20
|
+
# em.add_timer(1) do
|
21
|
+
# publisher.publish(channel, sent)
|
22
|
+
# end
|
23
|
+
# end
|
24
24
|
|
25
|
-
|
26
|
-
|
25
|
+
# recieved.should == sent
|
26
|
+
# end
|
27
27
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
28
|
+
# it "should track subscriptions" do
|
29
|
+
# lambda{
|
30
|
+
# em do
|
31
|
+
# consumer.subscribe_to channel
|
32
|
+
# consumer.subscribe_to another_channel
|
33
|
+
# em.add_timer(1){ em.stop }
|
34
|
+
# end
|
35
|
+
# }.should change{ consumer.subscriptions.size }.by(2)
|
36
|
+
# end
|
37
37
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
38
|
+
# it "should only allow one subscription per channel" do
|
39
|
+
# lambda{
|
40
|
+
# em do
|
41
|
+
# 3.times { consumer.subscribe_to channel }
|
42
|
+
# em.add_timer(1){ em.stop }
|
43
|
+
# end
|
44
|
+
# }.should change{ consumer.subscriptions.size }.by(1)
|
45
|
+
# end
|
46
46
|
|
47
|
-
|
48
|
-
|
47
|
+
# it "should unsubscribe from all channels" do
|
48
|
+
# subscribed_count, after_unsubscribe_count = 0, nil
|
49
49
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
50
|
+
# em do
|
51
|
+
# consumer.subscribe_to channel
|
52
|
+
# consumer.subscribe_to another_channel
|
53
|
+
# subscribed_count = consumer.subscriptions.size
|
54
|
+
# em.add_timer(1) do
|
55
|
+
# consumer.unsubscribe
|
56
|
+
# em.add_timer(1) do
|
57
|
+
# em.stop
|
58
|
+
# end
|
59
|
+
# end
|
60
|
+
# end
|
61
61
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
end
|
62
|
+
# subscribed_count.should == 2
|
63
|
+
# consumer.subscriptions.size.should == 0
|
64
|
+
# end
|
65
|
+
# end
|
66
|
+
# end
|
data/spec/spec_helper.rb
CHANGED
@@ -2,6 +2,7 @@ require 'logger'
|
|
2
2
|
require 'em-http'
|
3
3
|
require 'em-websocket-client'
|
4
4
|
|
5
|
+
# Skip logging if VERBOSE isn't set to true.
|
5
6
|
require 'firehose'
|
6
7
|
Firehose.logger = Logger.new('/dev/null') unless ENV['VERBOSE']
|
7
8
|
|
@@ -37,8 +38,4 @@ RSpec.configure do |config|
|
|
37
38
|
config.treat_symbols_as_metadata_keys_with_true_values = true
|
38
39
|
config.run_all_when_everything_filtered = true
|
39
40
|
config.filter_run :focus
|
40
|
-
config.before(:each) do
|
41
|
-
# For now, this resets the AMQP configuration between runs.
|
42
|
-
Firehose.reset!
|
43
|
-
end
|
44
41
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: firehose
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,11 +10,11 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2012-05-
|
13
|
+
date: 2012-05-09 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: eventmachine
|
17
|
-
requirement: &
|
17
|
+
requirement: &70097385746880 !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
20
20
|
- - ! '>='
|
@@ -22,21 +22,21 @@ dependencies:
|
|
22
22
|
version: 1.0.0.beta
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
|
-
version_requirements: *
|
25
|
+
version_requirements: *70097385746880
|
26
26
|
- !ruby/object:Gem::Dependency
|
27
|
-
name:
|
28
|
-
requirement: &
|
27
|
+
name: em-hiredis
|
28
|
+
requirement: &70097385746460 !ruby/object:Gem::Requirement
|
29
29
|
none: false
|
30
30
|
requirements:
|
31
31
|
- - ! '>='
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 0
|
33
|
+
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
|
-
version_requirements: *
|
36
|
+
version_requirements: *70097385746460
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
38
|
name: thin
|
39
|
-
requirement: &
|
39
|
+
requirement: &70097385746000 !ruby/object:Gem::Requirement
|
40
40
|
none: false
|
41
41
|
requirements:
|
42
42
|
- - ! '>='
|
@@ -44,10 +44,10 @@ dependencies:
|
|
44
44
|
version: '0'
|
45
45
|
type: :runtime
|
46
46
|
prerelease: false
|
47
|
-
version_requirements: *
|
47
|
+
version_requirements: *70097385746000
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
49
|
name: thor
|
50
|
-
requirement: &
|
50
|
+
requirement: &70097385745580 !ruby/object:Gem::Requirement
|
51
51
|
none: false
|
52
52
|
requirements:
|
53
53
|
- - ! '>='
|
@@ -55,10 +55,10 @@ dependencies:
|
|
55
55
|
version: '0'
|
56
56
|
type: :runtime
|
57
57
|
prerelease: false
|
58
|
-
version_requirements: *
|
58
|
+
version_requirements: *70097385745580
|
59
59
|
- !ruby/object:Gem::Dependency
|
60
60
|
name: faraday
|
61
|
-
requirement: &
|
61
|
+
requirement: &70097385745160 !ruby/object:Gem::Requirement
|
62
62
|
none: false
|
63
63
|
requirements:
|
64
64
|
- - ! '>='
|
@@ -66,10 +66,10 @@ dependencies:
|
|
66
66
|
version: '0'
|
67
67
|
type: :runtime
|
68
68
|
prerelease: false
|
69
|
-
version_requirements: *
|
69
|
+
version_requirements: *70097385745160
|
70
70
|
- !ruby/object:Gem::Dependency
|
71
71
|
name: websocket-rack
|
72
|
-
requirement: &
|
72
|
+
requirement: &70097385744740 !ruby/object:Gem::Requirement
|
73
73
|
none: false
|
74
74
|
requirements:
|
75
75
|
- - ! '>='
|
@@ -77,10 +77,10 @@ dependencies:
|
|
77
77
|
version: '0'
|
78
78
|
type: :runtime
|
79
79
|
prerelease: false
|
80
|
-
version_requirements: *
|
80
|
+
version_requirements: *70097385744740
|
81
81
|
- !ruby/object:Gem::Dependency
|
82
82
|
name: em-http-request
|
83
|
-
requirement: &
|
83
|
+
requirement: &70097385744240 !ruby/object:Gem::Requirement
|
84
84
|
none: false
|
85
85
|
requirements:
|
86
86
|
- - ~>
|
@@ -88,10 +88,10 @@ dependencies:
|
|
88
88
|
version: 1.0.0
|
89
89
|
type: :runtime
|
90
90
|
prerelease: false
|
91
|
-
version_requirements: *
|
91
|
+
version_requirements: *70097385744240
|
92
92
|
- !ruby/object:Gem::Dependency
|
93
93
|
name: rspec
|
94
|
-
requirement: &
|
94
|
+
requirement: &70097385743820 !ruby/object:Gem::Requirement
|
95
95
|
none: false
|
96
96
|
requirements:
|
97
97
|
- - ! '>='
|
@@ -99,10 +99,10 @@ dependencies:
|
|
99
99
|
version: '0'
|
100
100
|
type: :development
|
101
101
|
prerelease: false
|
102
|
-
version_requirements: *
|
102
|
+
version_requirements: *70097385743820
|
103
103
|
- !ruby/object:Gem::Dependency
|
104
104
|
name: webmock
|
105
|
-
requirement: &
|
105
|
+
requirement: &70097385743360 !ruby/object:Gem::Requirement
|
106
106
|
none: false
|
107
107
|
requirements:
|
108
108
|
- - ! '>='
|
@@ -110,10 +110,10 @@ dependencies:
|
|
110
110
|
version: '0'
|
111
111
|
type: :development
|
112
112
|
prerelease: false
|
113
|
-
version_requirements: *
|
113
|
+
version_requirements: *70097385743360
|
114
114
|
- !ruby/object:Gem::Dependency
|
115
115
|
name: guard-rspec
|
116
|
-
requirement: &
|
116
|
+
requirement: &70097385742940 !ruby/object:Gem::Requirement
|
117
117
|
none: false
|
118
118
|
requirements:
|
119
119
|
- - ! '>='
|
@@ -121,10 +121,10 @@ dependencies:
|
|
121
121
|
version: '0'
|
122
122
|
type: :development
|
123
123
|
prerelease: false
|
124
|
-
version_requirements: *
|
124
|
+
version_requirements: *70097385742940
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: guard-bundler
|
127
|
-
requirement: &
|
127
|
+
requirement: &70097385742520 !ruby/object:Gem::Requirement
|
128
128
|
none: false
|
129
129
|
requirements:
|
130
130
|
- - ! '>='
|
@@ -132,10 +132,10 @@ dependencies:
|
|
132
132
|
version: '0'
|
133
133
|
type: :development
|
134
134
|
prerelease: false
|
135
|
-
version_requirements: *
|
135
|
+
version_requirements: *70097385742520
|
136
136
|
- !ruby/object:Gem::Dependency
|
137
137
|
name: guard-coffeescript
|
138
|
-
requirement: &
|
138
|
+
requirement: &70097385742100 !ruby/object:Gem::Requirement
|
139
139
|
none: false
|
140
140
|
requirements:
|
141
141
|
- - ! '>='
|
@@ -143,10 +143,10 @@ dependencies:
|
|
143
143
|
version: '0'
|
144
144
|
type: :development
|
145
145
|
prerelease: false
|
146
|
-
version_requirements: *
|
146
|
+
version_requirements: *70097385742100
|
147
147
|
- !ruby/object:Gem::Dependency
|
148
148
|
name: em-websocket-client
|
149
|
-
requirement: &
|
149
|
+
requirement: &70097385741680 !ruby/object:Gem::Requirement
|
150
150
|
none: false
|
151
151
|
requirements:
|
152
152
|
- - ! '>='
|
@@ -154,7 +154,7 @@ dependencies:
|
|
154
154
|
version: '0'
|
155
155
|
type: :development
|
156
156
|
prerelease: false
|
157
|
-
version_requirements: *
|
157
|
+
version_requirements: *70097385741680
|
158
158
|
description: Firehose is a realtime web application toolkit for building realtime
|
159
159
|
Ruby web applications.
|
160
160
|
email:
|
@@ -184,16 +184,13 @@ files:
|
|
184
184
|
- lib/assets/javascripts/firehose/transport.js.coffee
|
185
185
|
- lib/assets/javascripts/firehose/web_socket.js.coffee
|
186
186
|
- lib/firehose.rb
|
187
|
-
- lib/firehose/broker.rb
|
188
187
|
- lib/firehose/cli.rb
|
189
|
-
- lib/firehose/consumer.rb
|
190
188
|
- lib/firehose/default.rb
|
191
189
|
- lib/firehose/producer.rb
|
192
190
|
- lib/firehose/publisher.rb
|
193
191
|
- lib/firehose/rack.rb
|
194
192
|
- lib/firehose/subscription.rb
|
195
193
|
- lib/firehose/version.rb
|
196
|
-
- spec/integrations/amqp_resources_spec.rb
|
197
194
|
- spec/integrations/thin_spec.rb
|
198
195
|
- spec/lib/broker_spec.rb
|
199
196
|
- spec/lib/consumer_spec.rb
|
@@ -225,7 +222,6 @@ signing_key:
|
|
225
222
|
specification_version: 3
|
226
223
|
summary: Build realtime Ruby web applications
|
227
224
|
test_files:
|
228
|
-
- spec/integrations/amqp_resources_spec.rb
|
229
225
|
- spec/integrations/thin_spec.rb
|
230
226
|
- spec/lib/broker_spec.rb
|
231
227
|
- spec/lib/consumer_spec.rb
|
data/lib/firehose/broker.rb
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
module Firehose
|
2
|
-
# TODO add support to broker for publishing, then abstract this out into a backend. A
|
3
|
-
# broker will eventually be passed into a web server front-end to serve up the web requests.
|
4
|
-
class Broker
|
5
|
-
def consumers
|
6
|
-
@consumers ||= Hash.new do |consumers, consumer_id|
|
7
|
-
consumer = Firehose::Consumer.new(consumer_id)
|
8
|
-
consumer.on_unsubscribe do
|
9
|
-
consumers.delete consumer_id
|
10
|
-
end
|
11
|
-
consumers[consumer_id] = consumer
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
# Don't like the [] syntax to get at consumers? No worries mate!
|
16
|
-
def consumer(consumer_id)
|
17
|
-
consumers[consumer_id]
|
18
|
-
end
|
19
|
-
|
20
|
-
# Gracefully unsubscribe all of the consumers and get rid of them from the consumers
|
21
|
-
# collection.
|
22
|
-
def stop
|
23
|
-
consumers.values.each(&:unsubscribe)
|
24
|
-
end
|
25
|
-
|
26
|
-
# Returns a hash of the connected consumers with the number of their subscriptions
|
27
|
-
def stats
|
28
|
-
consumers.inject({}) do |memo, (consumer_id, consumer)|
|
29
|
-
memo[consumer_id] = { 'subscriptions' => consumer.subscriptions.keys }
|
30
|
-
memo
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
data/lib/firehose/consumer.rb
DELETED
@@ -1,49 +0,0 @@
|
|
1
|
-
module Firehose
|
2
|
-
class Consumer
|
3
|
-
# Unique identifier for a consumer. Note that a consumer does not map directly to
|
4
|
-
# a user_id. In a web browser world, you might have a user with multiple tabs open,
|
5
|
-
# so you'll went to send each users tab a seperate message stream. Consider a convention
|
6
|
-
# such as :user_id-:guid for your application.
|
7
|
-
attr_reader :guid
|
8
|
-
|
9
|
-
def initialize(guid = self.class.next_guid)
|
10
|
-
@guid = guid
|
11
|
-
end
|
12
|
-
|
13
|
-
# Create a subscription and subscribe to a channel.
|
14
|
-
def subscribe_to(channel, &block)
|
15
|
-
subscriptions[channel].subscribe(&block)
|
16
|
-
end
|
17
|
-
|
18
|
-
# Active subscriptions to the backend.
|
19
|
-
def subscriptions
|
20
|
-
@subscriptions ||= Hash.new do |subscriptions, channel|
|
21
|
-
# Setup the hash to generate subscriptions that can delete themselves from
|
22
|
-
# their own collection on an unsubscription event.
|
23
|
-
subscription = Subscription.new(self, channel)
|
24
|
-
subscription.on_unsubscribe do
|
25
|
-
# Remove the subscription from the consumer.subscriptions
|
26
|
-
# list when unsubscribe.
|
27
|
-
subscriptions.delete channel
|
28
|
-
end
|
29
|
-
subscriptions[channel] = subscription
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
# Unsubscribe from all subscriptions.
|
34
|
-
def unsubscribe
|
35
|
-
subscriptions.values.each(&:unsubscribe)
|
36
|
-
@on_unsubscribe.call(self) if @on_unsubscribe
|
37
|
-
end
|
38
|
-
|
39
|
-
# Define callback for when unsubscribe is called from the consumer.
|
40
|
-
def on_unsubscribe(&block)
|
41
|
-
@on_unsubscribe = block
|
42
|
-
end
|
43
|
-
|
44
|
-
protected
|
45
|
-
def self.next_guid
|
46
|
-
rand(999_999_999_999).to_s
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
@@ -1,51 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe "Firehose amqp resources" do
|
4
|
-
|
5
|
-
let(:channel) { "/resource-test-#{Time.now.to_i}" }
|
6
|
-
let(:consumer) { Firehose::Consumer.new }
|
7
|
-
|
8
|
-
it "should clean up exchanges and queues" do
|
9
|
-
sent, received = 'howdy!', nil
|
10
|
-
|
11
|
-
before_exchange_count = `rabbitmqctl list_exchanges`.lines.count
|
12
|
-
before_queue_count = `rabbitmqctl list_queues`.lines.count
|
13
|
-
|
14
|
-
during_exchange_count = nil
|
15
|
-
during_queue_count = nil
|
16
|
-
|
17
|
-
EM.run do
|
18
|
-
# Kill test if it runs longer than 5s
|
19
|
-
EM.add_timer(5) { EM.stop }
|
20
|
-
|
21
|
-
subscription = Firehose::Subscription.new(consumer, channel)
|
22
|
-
subscription.ttl = 1
|
23
|
-
|
24
|
-
subscription.subscribe do |payload|
|
25
|
-
received = payload
|
26
|
-
subscription.unsubscribe
|
27
|
-
|
28
|
-
during_exchange_count = `rabbitmqctl list_exchanges`.lines.count
|
29
|
-
during_queue_count = `rabbitmqctl list_queues`.lines.count
|
30
|
-
|
31
|
-
# I wait 1 second before killing em so that unsubscribe
|
32
|
-
# can talk to AMQP before the whole thing dies.
|
33
|
-
EM.add_timer(1){ EM.stop }
|
34
|
-
end
|
35
|
-
|
36
|
-
# Let the subscriber subscribe before publishing messages.
|
37
|
-
EM.add_timer(1){ Firehose::Publisher.new.publish(channel, sent) }
|
38
|
-
end
|
39
|
-
|
40
|
-
after_exchange_count = `rabbitmqctl list_exchanges`.lines.count
|
41
|
-
after_queue_count = `rabbitmqctl list_queues`.lines.count
|
42
|
-
|
43
|
-
received.should == sent
|
44
|
-
|
45
|
-
after_exchange_count.should == before_exchange_count
|
46
|
-
after_queue_count.should == before_queue_count
|
47
|
-
|
48
|
-
during_exchange_count.should == before_exchange_count + 1
|
49
|
-
during_queue_count.should == before_queue_count + 1
|
50
|
-
end
|
51
|
-
end
|