firehose 0.1.1 → 0.2.alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.env.sample +10 -0
- data/.gitignore +2 -0
- data/Procfile +1 -1
- data/README.md +117 -11
- data/config/rainbows.rb +20 -0
- data/firehose.gemspec +9 -6
- data/lib/assets/flash/firehose/WebSocketMain.swf +0 -0
- data/lib/assets/javascripts/firehose.js.coffee +4 -1
- data/lib/assets/javascripts/firehose/consumer.js.coffee +3 -11
- data/lib/assets/javascripts/firehose/lib/jquery.cors.headers.js.coffee +16 -0
- data/lib/assets/javascripts/firehose/lib/swfobject.js +4 -0
- data/lib/assets/javascripts/firehose/lib/web_socket.js +389 -0
- data/lib/assets/javascripts/firehose/long_poll.js.coffee +42 -39
- data/lib/assets/javascripts/firehose/transport.js.coffee +1 -1
- data/lib/assets/javascripts/firehose/web_socket.js.coffee +8 -14
- data/lib/firehose.rb +12 -17
- data/lib/firehose/channel.rb +84 -0
- data/lib/firehose/cli.rb +57 -13
- data/lib/firehose/client.rb +92 -0
- data/lib/firehose/default.rb +2 -2
- data/lib/firehose/logging.rb +35 -0
- data/lib/firehose/producer.rb +1 -0
- data/lib/firehose/publisher.rb +56 -4
- data/lib/firehose/rack.rb +37 -120
- data/lib/firehose/rack/consumer_app.rb +143 -0
- data/lib/firehose/rack/ping_app.rb +84 -0
- data/lib/firehose/rack/publisher_app.rb +40 -0
- data/lib/firehose/server.rb +48 -0
- data/lib/firehose/subscriber.rb +54 -0
- data/lib/firehose/swf_policy_request.rb +23 -0
- data/lib/firehose/version.rb +2 -2
- data/lib/rainbows_em_swf_policy.rb +33 -0
- data/lib/thin_em_swf_policy.rb +26 -0
- data/spec/integrations/integration_test_helper.rb +16 -0
- data/spec/integrations/rainbows_spec.rb +7 -0
- data/spec/integrations/shared_examples.rb +111 -0
- data/spec/integrations/thin_spec.rb +5 -79
- data/spec/lib/channel_spec.rb +164 -0
- data/spec/lib/client_spec.rb +9 -0
- data/spec/lib/default_spec.rb +2 -2
- data/spec/lib/publisher_spec.rb +82 -0
- data/spec/lib/rack/consumer_app_spec.rb +11 -0
- data/spec/lib/rack/ping_app_spec.rb +28 -0
- data/spec/lib/rack/publisher_app_spec.rb +26 -0
- data/spec/lib/subscriber_spec.rb +69 -0
- data/spec/spec_helper.rb +49 -8
- metadata +114 -45
- data/config.ru +0 -6
- data/lib/firehose/subscription.rb +0 -77
@@ -0,0 +1,40 @@
|
|
1
|
+
module Firehose
|
2
|
+
module Rack
|
3
|
+
class PublisherApp
|
4
|
+
include Firehose::Rack::Helpers
|
5
|
+
|
6
|
+
def call(env)
|
7
|
+
req = env['parsed_request'] ||= ::Rack::Request.new(env)
|
8
|
+
path = req.path
|
9
|
+
method = req.request_method
|
10
|
+
|
11
|
+
if method == 'PUT'
|
12
|
+
EM.next_tick do
|
13
|
+
body = env['rack.input'].read
|
14
|
+
Firehose.logger.debug "HTTP published `#{body}` to `#{path}`"
|
15
|
+
publisher.publish(path, body).callback do
|
16
|
+
env['async.callback'].call [202, {'Content-Type' => 'text/plain', 'Content-Length' => '0'}, []]
|
17
|
+
env['async.callback'].call response(202, '', 'Content-Type' => 'text/plain')
|
18
|
+
end.errback do |e|
|
19
|
+
Firehose.logger.debug "Error publishing: #{e.inspect}"
|
20
|
+
env['async.callback'].call response(500, 'Error when trying to publish', 'Content-Type' => 'text/plain')
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# Tell the web server that this will be an async response.
|
25
|
+
ASYNC_RESPONSE
|
26
|
+
else
|
27
|
+
Firehose.logger.debug "HTTP #{method} not supported"
|
28
|
+
msg = "#{method} not supported."
|
29
|
+
[501, {'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s}, [msg]]
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
private
|
35
|
+
def publisher
|
36
|
+
@publisher ||= Firehose::Publisher.new
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'faye/websocket'
|
2
|
+
|
3
|
+
module Firehose
|
4
|
+
class Server
|
5
|
+
def initialize(opts={})
|
6
|
+
@port = opts[:port] || Firehose::Default::URI.port
|
7
|
+
@host = opts[:host] || Firehose::Default::URI.host
|
8
|
+
@server = opts[:server] || :rainbows
|
9
|
+
|
10
|
+
Firehose.logger.info "Starting #{Firehose::VERSION} '#{Firehose::CODENAME}', in #{ENV['RACK_ENV']}"
|
11
|
+
end
|
12
|
+
|
13
|
+
def start
|
14
|
+
self.send("start_#{@server}")
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
def start_rainbows
|
19
|
+
require 'rainbows'
|
20
|
+
Faye::WebSocket.load_adapter('rainbows')
|
21
|
+
|
22
|
+
rackup = Unicorn::Configurator::RACKUP
|
23
|
+
rackup[:port] = @port if @port
|
24
|
+
rackup[:host] = @host if @host
|
25
|
+
rackup[:set_listener] = true
|
26
|
+
opts = rackup[:options]
|
27
|
+
opts[:config_file] = File.expand_path('../../../config/rainbows.rb', __FILE__)
|
28
|
+
|
29
|
+
server = Rainbows::HttpServer.new(Firehose::Rack::App.new, opts)
|
30
|
+
server.start.join
|
31
|
+
end
|
32
|
+
|
33
|
+
def start_thin
|
34
|
+
require 'thin'
|
35
|
+
require 'thin_em_swf_policy' if ENV['RACK_ENV'] == 'development'
|
36
|
+
|
37
|
+
Faye::WebSocket.load_adapter('thin')
|
38
|
+
|
39
|
+
# TODO: See if we can just set Thin to use Firehose.logger instead of
|
40
|
+
# printing out messages by itself.
|
41
|
+
Thin::Logging.silent = true if Firehose.logger.level == Logger::ERROR
|
42
|
+
|
43
|
+
server = Thin::Server.new(@host, @port) do
|
44
|
+
run Firehose::Rack::App.new
|
45
|
+
end.start
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Firehose
|
2
|
+
# Setups a connetion to Redis to listen for new resources...
|
3
|
+
class Subscriber
|
4
|
+
attr_reader :redis
|
5
|
+
|
6
|
+
def initialize(redis)
|
7
|
+
@redis = redis
|
8
|
+
|
9
|
+
# TODO: Instead of just raising an exception, it would probably be better
|
10
|
+
# for the errback to set some sort of 'disconnected' state. Then
|
11
|
+
# whenever a deferrable was 'subscribed' we could instantly fail
|
12
|
+
# the deferrable with whatever connection error we had.
|
13
|
+
# An alternative which would have a similar result would be to
|
14
|
+
# subscribe lazily (i.e. not until we have a deferrable to subscribe).
|
15
|
+
# Then, if connecting failed, it'd be super easy to fail the deferrable
|
16
|
+
# with the same error.
|
17
|
+
# The final goal is to allow the failed deferrable bubble back up
|
18
|
+
# so we can send back a nice, clean 500 error to the client.
|
19
|
+
redis.subscribe('firehose:channel_updates').
|
20
|
+
errback{|e| EM.next_tick { raise e } }.
|
21
|
+
callback { Firehose.logger.debug "Redis subscribed to `firehose:channel_updates`" }
|
22
|
+
redis.on(:message) do |_, payload|
|
23
|
+
channel_key, sequence, message = Firehose::Publisher.from_payload(payload)
|
24
|
+
|
25
|
+
if deferrables = subscriptions.delete(channel_key)
|
26
|
+
Firehose.logger.debug "Redis notifying #{deferrables.count} deferrable(s) at `#{channel_key}` with sequence `#{sequence}` and message `#{message}`"
|
27
|
+
deferrables.each do |deferrable|
|
28
|
+
Firehose.logger.debug "Sending message #{message} and sequence #{sequence} to client from subscriber"
|
29
|
+
deferrable.succeed message, sequence.to_i
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def subscribe(channel_key, deferrable)
|
36
|
+
subscriptions[channel_key].push deferrable
|
37
|
+
end
|
38
|
+
|
39
|
+
def unsubscribe(channel_key, deferrable)
|
40
|
+
subscriptions[channel_key].delete deferrable
|
41
|
+
subscriptions.delete(channel_key) if subscriptions[channel_key].empty?
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
private
|
46
|
+
def subscriptions
|
47
|
+
@subscriptions ||= Hash.new{|h,k| h[k] = []}
|
48
|
+
end
|
49
|
+
|
50
|
+
def key(*segments)
|
51
|
+
segments.unshift(:firehose).join(':')
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Firehose
|
2
|
+
module SwfPolicyRequest
|
3
|
+
|
4
|
+
# Borrowed from: https://github.com/igrigorik/em-websocket/blob/3e7f7d7760cc23b9d1d34fc1c17bab4423b5d11a/lib/em-websocket/connection.rb#L104
|
5
|
+
def handle_swf_policy_request(data)
|
6
|
+
if data =~ /\A<policy-file-request\s*\/>/
|
7
|
+
Firehose.logger.debug "Received SWF Policy request: #{data.inspect}"
|
8
|
+
send_data policy
|
9
|
+
close_connection_after_writing
|
10
|
+
true
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def policy
|
15
|
+
<<-EOS
|
16
|
+
<?xml version="1.0"?>
|
17
|
+
<cross-domain-policy>
|
18
|
+
<allow-access-from domain="*" to-ports="*"/>
|
19
|
+
</cross-domain-policy>
|
20
|
+
EOS
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/firehose/version.rb
CHANGED
@@ -0,0 +1,33 @@
|
|
1
|
+
# This file file monkeypatches Rainbows! to return a proper SWF policy file.
|
2
|
+
# Enable this with something like this in your config/rainbows.rb file:
|
3
|
+
#
|
4
|
+
# after_fork do |server, worker|
|
5
|
+
# require 'rainbows_em_swf_policy'
|
6
|
+
# end if ENV['RACK_ENV'] == 'development'
|
7
|
+
#
|
8
|
+
# You should only use this in development. It has not been well tested in a
|
9
|
+
# production environment.
|
10
|
+
#
|
11
|
+
# NOTE: This only works if you are using Rainbows! with EventMachine.
|
12
|
+
#
|
13
|
+
# Some helpful links:
|
14
|
+
# http://unicorn.bogomips.org/Unicorn/Configurator.html
|
15
|
+
# http://www.adobe.com/devnet/flashplayer/articles/socket_policy_files.html
|
16
|
+
# http://blog.vokle.com/index.php/2009/06/10/dealing-with-adobe-and-serving-socket-policy-servers-via-nginx-and-10-lines-of-code/
|
17
|
+
|
18
|
+
require 'rainbows'
|
19
|
+
# Ensure the class already exists so we are overwriting it.
|
20
|
+
Rainbows::EventMachine::Client
|
21
|
+
|
22
|
+
class Rainbows::EventMachine::Client
|
23
|
+
include Firehose::SwfPolicyRequest
|
24
|
+
alias_method :receive_data_without_swf_policy, :receive_data
|
25
|
+
# Borrowed from: https://github.com/igrigorik/em-websocket/blob/3e7f7d7760cc23b9d1d34fc1c17bab4423b5d11a/lib/em-websocket/connection.rb#L104
|
26
|
+
def receive_data(data)
|
27
|
+
if handle_swf_policy_request(data)
|
28
|
+
return false
|
29
|
+
else
|
30
|
+
receive_data_without_swf_policy(data)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# This file file monkeypatches Thin to return a proper SWF policy file.
|
2
|
+
#
|
3
|
+
# You should only use this in development. It has not been well tested in a
|
4
|
+
# production environment.
|
5
|
+
#
|
6
|
+
# NOTE: This only works if you are using Thin with EventMachine.
|
7
|
+
#
|
8
|
+
# Some helpful links:
|
9
|
+
# http://www.adobe.com/devnet/flashplayer/articles/socket_policy_files.html
|
10
|
+
# http://blog.vokle.com/index.php/2009/06/10/dealing-with-adobe-and-serving-socket-policy-servers-via-nginx-and-10-lines-of-code/
|
11
|
+
|
12
|
+
require 'thin'
|
13
|
+
# Ensure the class already exists so we are overwriting it.
|
14
|
+
Thin::Connection
|
15
|
+
|
16
|
+
class Thin::Connection
|
17
|
+
include Firehose::SwfPolicyRequest
|
18
|
+
alias_method :receive_data_without_swf_policy, :receive_data
|
19
|
+
def receive_data(data)
|
20
|
+
if handle_swf_policy_request(data)
|
21
|
+
return false
|
22
|
+
else
|
23
|
+
receive_data_without_swf_policy(data)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module IntegrationTestHelper
|
4
|
+
def start_server
|
5
|
+
@server_pid = fork do
|
6
|
+
Firehose::Server.new(:server => server, :port => uri.port).start
|
7
|
+
end
|
8
|
+
|
9
|
+
# Need to give the server a chance to boot up.
|
10
|
+
sleep 1
|
11
|
+
end
|
12
|
+
|
13
|
+
def stop_server
|
14
|
+
Process.kill 'INT', @server_pid
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'integrations/integration_test_helper'
|
3
|
+
|
4
|
+
shared_examples_for 'Firehose::Rack::App' do
|
5
|
+
include EM::TestHelper
|
6
|
+
include IntegrationTestHelper
|
7
|
+
|
8
|
+
before(:all) do
|
9
|
+
Firehose::Producer.adapter = :em_http
|
10
|
+
start_server
|
11
|
+
end
|
12
|
+
|
13
|
+
after(:all) do
|
14
|
+
Firehose::Producer.adapter = nil
|
15
|
+
stop_server
|
16
|
+
end
|
17
|
+
|
18
|
+
before(:each) { WebMock.disable! }
|
19
|
+
after(:each) { WebMock.enable! }
|
20
|
+
|
21
|
+
let(:app) { Firehose::Rack::App.new }
|
22
|
+
let(:messages) { (1..2000).map{|n| "msg-#{n}" } }
|
23
|
+
let(:channel) { "/firehose/integration/#{Time.now.to_i}" }
|
24
|
+
let(:uri) { Firehose::Default::URI }
|
25
|
+
let(:http_url) { "http://#{uri.host}:#{uri.port}#{channel}" }
|
26
|
+
let(:ws_url) { "ws://#{uri.host}:#{uri.port}#{channel}" }
|
27
|
+
|
28
|
+
it "should pub-sub http and websockets" do
|
29
|
+
# Setup variables that we'll use after we turn off EM to validate our
|
30
|
+
# test assertions.
|
31
|
+
outgoing, received = messages.dup, Hash.new{|h,k| h[k] = []}
|
32
|
+
|
33
|
+
# Our WS and Http clients call this when they have received their messages to determine
|
34
|
+
# when to turn off EM and make the test assertion at the very bottom.
|
35
|
+
succeed = Proc.new do
|
36
|
+
# TODO: For some weird reason the `add_timer` call causes up to 20 seconds of delay after
|
37
|
+
# the test finishes running. However, without it the test will randomly fail with a
|
38
|
+
# "Redis disconnected" error.
|
39
|
+
em.add_timer(1) { em.stop } if received.values.all?{|arr| arr.size == messages.size }
|
40
|
+
end
|
41
|
+
|
42
|
+
# Setup a publisher
|
43
|
+
publish = Proc.new do
|
44
|
+
Firehose::Producer.new.publish(outgoing.shift).to(channel) do
|
45
|
+
# The random timer ensures that sometimes the clients will be behind
|
46
|
+
# and sometimes they will be caught up.
|
47
|
+
EM::add_timer(rand*0.005) { publish.call } unless outgoing.empty?
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Lets have an HTTP Long poll client
|
52
|
+
http_long_poll = Proc.new do |cid, last_sequence|
|
53
|
+
http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => last_sequence})
|
54
|
+
http.errback { em.stop }
|
55
|
+
http.callback do
|
56
|
+
received[cid] << http.response
|
57
|
+
if received[cid].size < messages.size
|
58
|
+
# Add some jitter so the clients aren't syncronized
|
59
|
+
EM::add_timer(rand*0.001) { http_long_poll.call cid, http.response_header['pragma'] }
|
60
|
+
else
|
61
|
+
succeed.call cid
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# And test a web socket client too, at the same time.
|
67
|
+
websocket = Proc.new do |cid|
|
68
|
+
ws = Faye::WebSocket::Client.new(ws_url)
|
69
|
+
ws.onmessage = lambda do |event|
|
70
|
+
received[cid] << event.data
|
71
|
+
succeed.call cid unless received[cid].size < messages.size
|
72
|
+
end
|
73
|
+
|
74
|
+
ws.onclose = lambda do |event|
|
75
|
+
ws = nil
|
76
|
+
end
|
77
|
+
|
78
|
+
ws.onerror { |e| raise 'ws failed' + "\n" + e.inspect }
|
79
|
+
end
|
80
|
+
|
81
|
+
# Great, we have all the pieces in order, lets run this thing in the reactor.
|
82
|
+
em 60 do
|
83
|
+
# Start the clients.
|
84
|
+
websocket.call(1)
|
85
|
+
websocket.call(2)
|
86
|
+
http_long_poll.call(3)
|
87
|
+
http_long_poll.call(4)
|
88
|
+
|
89
|
+
# Wait a sec to let our clients set up.
|
90
|
+
em.add_timer(1){ publish.call }
|
91
|
+
end
|
92
|
+
|
93
|
+
# When EM stops, these assertions will be made.
|
94
|
+
received.size.should == 4
|
95
|
+
received.values.each do |arr|
|
96
|
+
arr.should == messages
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
|
101
|
+
it "should return 400 error for long-polling when using http long polling and sequence header is < 0" do
|
102
|
+
em 5 do
|
103
|
+
http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => -1})
|
104
|
+
http.errback { |e| raise e.inspect }
|
105
|
+
http.callback do
|
106
|
+
http.response_header.status.should == 400
|
107
|
+
em.stop
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -1,81 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
+
require 'integrations/shared_examples'
|
2
3
|
|
3
|
-
describe
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
Firehose::Producer.adapter = :em_http
|
8
|
-
end
|
9
|
-
|
10
|
-
after(:all) do
|
11
|
-
Firehose::Producer.adapter = nil
|
12
|
-
end
|
13
|
-
|
14
|
-
let(:app) { Firehose::Rack::App.new }
|
15
|
-
let(:messages) { (1..2000).map(&:to_s) }
|
16
|
-
let(:channel) { "/firehose/integration/#{Time.now.to_i}" }
|
17
|
-
let(:uri) { Firehose::Default::URI }
|
18
|
-
let(:http_url) { "http://#{uri.host}:#{uri.port}#{channel}" }
|
19
|
-
let(:ws_url) { "ws://#{uri.host}:#{uri.port}#{channel}" }
|
20
|
-
|
21
|
-
it "should pub-sub http and websockets" do
|
22
|
-
# Setup variables that we'll use after we turn off EM to validate our
|
23
|
-
# test assertions.
|
24
|
-
outgoing, received_http, received_ws = messages.dup, [], []
|
25
|
-
|
26
|
-
# Our WS and Http clients call this when they have received their messages to determine
|
27
|
-
# when to turn off EM and make the test assertion at the very bottom.
|
28
|
-
succeed = Proc.new do
|
29
|
-
em.stop if received_http.size == messages.size and received_ws.size == messages.size
|
30
|
-
end
|
31
|
-
|
32
|
-
# Setup a publisher
|
33
|
-
publish = Proc.new do
|
34
|
-
Firehose::Producer.new.publish(outgoing.pop).to(channel) do
|
35
|
-
publish.call unless outgoing.empty?
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
# Lets have an HTTP Long poll client
|
40
|
-
http_long_poll = Proc.new do
|
41
|
-
http = EM::HttpRequest.new(http_url).get(:query => {'cid' => 'alpha'})
|
42
|
-
http.errback { em.stop }
|
43
|
-
http.callback do
|
44
|
-
received_http << http.response
|
45
|
-
if received_http.size < messages.size
|
46
|
-
http_long_poll.call
|
47
|
-
else
|
48
|
-
succeed.call
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
# And test a web socket client too, at the same time.
|
54
|
-
websocket = Proc.new do
|
55
|
-
ws = EventMachine::WebSocketClient.connect("#{ws_url}?cid=bravo")
|
56
|
-
ws.errback { em.stop }
|
57
|
-
ws.stream do |msg|
|
58
|
-
received_ws << msg
|
59
|
-
succeed.call unless received_ws.size < messages.size
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
# Great, we have all the pieces in order, lets run this thing in the reactor.
|
64
|
-
em do
|
65
|
-
# Start the server
|
66
|
-
server = ::Thin::Server.new('0.0.0.0', uri.port, app)
|
67
|
-
server.start
|
68
|
-
|
69
|
-
# Start the http_long_pollr.
|
70
|
-
http_long_poll.call
|
71
|
-
websocket.call
|
72
|
-
|
73
|
-
# Wait a sec to let our http_long_poll setup.
|
74
|
-
em.add_timer(1){ publish.call }
|
75
|
-
end
|
76
|
-
|
77
|
-
# When EM stops, these assertions will be made.
|
78
|
-
received_ws.should =~ messages
|
79
|
-
received_http.should =~ messages
|
80
|
-
end
|
81
|
-
end
|
4
|
+
describe "thin" do
|
5
|
+
let(:server) { :thin }
|
6
|
+
it_behaves_like 'Firehose::Rack::App'
|
7
|
+
end
|