firehose 0.1.1 → 0.2.alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/.env.sample +10 -0
  2. data/.gitignore +2 -0
  3. data/Procfile +1 -1
  4. data/README.md +117 -11
  5. data/config/rainbows.rb +20 -0
  6. data/firehose.gemspec +9 -6
  7. data/lib/assets/flash/firehose/WebSocketMain.swf +0 -0
  8. data/lib/assets/javascripts/firehose.js.coffee +4 -1
  9. data/lib/assets/javascripts/firehose/consumer.js.coffee +3 -11
  10. data/lib/assets/javascripts/firehose/lib/jquery.cors.headers.js.coffee +16 -0
  11. data/lib/assets/javascripts/firehose/lib/swfobject.js +4 -0
  12. data/lib/assets/javascripts/firehose/lib/web_socket.js +389 -0
  13. data/lib/assets/javascripts/firehose/long_poll.js.coffee +42 -39
  14. data/lib/assets/javascripts/firehose/transport.js.coffee +1 -1
  15. data/lib/assets/javascripts/firehose/web_socket.js.coffee +8 -14
  16. data/lib/firehose.rb +12 -17
  17. data/lib/firehose/channel.rb +84 -0
  18. data/lib/firehose/cli.rb +57 -13
  19. data/lib/firehose/client.rb +92 -0
  20. data/lib/firehose/default.rb +2 -2
  21. data/lib/firehose/logging.rb +35 -0
  22. data/lib/firehose/producer.rb +1 -0
  23. data/lib/firehose/publisher.rb +56 -4
  24. data/lib/firehose/rack.rb +37 -120
  25. data/lib/firehose/rack/consumer_app.rb +143 -0
  26. data/lib/firehose/rack/ping_app.rb +84 -0
  27. data/lib/firehose/rack/publisher_app.rb +40 -0
  28. data/lib/firehose/server.rb +48 -0
  29. data/lib/firehose/subscriber.rb +54 -0
  30. data/lib/firehose/swf_policy_request.rb +23 -0
  31. data/lib/firehose/version.rb +2 -2
  32. data/lib/rainbows_em_swf_policy.rb +33 -0
  33. data/lib/thin_em_swf_policy.rb +26 -0
  34. data/spec/integrations/integration_test_helper.rb +16 -0
  35. data/spec/integrations/rainbows_spec.rb +7 -0
  36. data/spec/integrations/shared_examples.rb +111 -0
  37. data/spec/integrations/thin_spec.rb +5 -79
  38. data/spec/lib/channel_spec.rb +164 -0
  39. data/spec/lib/client_spec.rb +9 -0
  40. data/spec/lib/default_spec.rb +2 -2
  41. data/spec/lib/publisher_spec.rb +82 -0
  42. data/spec/lib/rack/consumer_app_spec.rb +11 -0
  43. data/spec/lib/rack/ping_app_spec.rb +28 -0
  44. data/spec/lib/rack/publisher_app_spec.rb +26 -0
  45. data/spec/lib/subscriber_spec.rb +69 -0
  46. data/spec/spec_helper.rb +49 -8
  47. metadata +114 -45
  48. data/config.ru +0 -6
  49. data/lib/firehose/subscription.rb +0 -77
@@ -0,0 +1,40 @@
1
+ module Firehose
2
+ module Rack
3
+ class PublisherApp
4
+ include Firehose::Rack::Helpers
5
+
6
+ def call(env)
7
+ req = env['parsed_request'] ||= ::Rack::Request.new(env)
8
+ path = req.path
9
+ method = req.request_method
10
+
11
+ if method == 'PUT'
12
+ EM.next_tick do
13
+ body = env['rack.input'].read
14
+ Firehose.logger.debug "HTTP published `#{body}` to `#{path}`"
15
+ publisher.publish(path, body).callback do
16
+ env['async.callback'].call [202, {'Content-Type' => 'text/plain', 'Content-Length' => '0'}, []]
17
+ env['async.callback'].call response(202, '', 'Content-Type' => 'text/plain')
18
+ end.errback do |e|
19
+ Firehose.logger.debug "Error publishing: #{e.inspect}"
20
+ env['async.callback'].call response(500, 'Error when trying to publish', 'Content-Type' => 'text/plain')
21
+ end
22
+ end
23
+
24
+ # Tell the web server that this will be an async response.
25
+ ASYNC_RESPONSE
26
+ else
27
+ Firehose.logger.debug "HTTP #{method} not supported"
28
+ msg = "#{method} not supported."
29
+ [501, {'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s}, [msg]]
30
+ end
31
+ end
32
+
33
+
34
+ private
35
+ def publisher
36
+ @publisher ||= Firehose::Publisher.new
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,48 @@
1
+ require 'faye/websocket'
2
+
3
+ module Firehose
4
+ class Server
5
+ def initialize(opts={})
6
+ @port = opts[:port] || Firehose::Default::URI.port
7
+ @host = opts[:host] || Firehose::Default::URI.host
8
+ @server = opts[:server] || :rainbows
9
+
10
+ Firehose.logger.info "Starting #{Firehose::VERSION} '#{Firehose::CODENAME}', in #{ENV['RACK_ENV']}"
11
+ end
12
+
13
+ def start
14
+ self.send("start_#{@server}")
15
+ end
16
+
17
+ private
18
+ def start_rainbows
19
+ require 'rainbows'
20
+ Faye::WebSocket.load_adapter('rainbows')
21
+
22
+ rackup = Unicorn::Configurator::RACKUP
23
+ rackup[:port] = @port if @port
24
+ rackup[:host] = @host if @host
25
+ rackup[:set_listener] = true
26
+ opts = rackup[:options]
27
+ opts[:config_file] = File.expand_path('../../../config/rainbows.rb', __FILE__)
28
+
29
+ server = Rainbows::HttpServer.new(Firehose::Rack::App.new, opts)
30
+ server.start.join
31
+ end
32
+
33
+ def start_thin
34
+ require 'thin'
35
+ require 'thin_em_swf_policy' if ENV['RACK_ENV'] == 'development'
36
+
37
+ Faye::WebSocket.load_adapter('thin')
38
+
39
+ # TODO: See if we can just set Thin to use Firehose.logger instead of
40
+ # printing out messages by itself.
41
+ Thin::Logging.silent = true if Firehose.logger.level == Logger::ERROR
42
+
43
+ server = Thin::Server.new(@host, @port) do
44
+ run Firehose::Rack::App.new
45
+ end.start
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,54 @@
1
+ module Firehose
2
+ # Setups a connetion to Redis to listen for new resources...
3
+ class Subscriber
4
+ attr_reader :redis
5
+
6
+ def initialize(redis)
7
+ @redis = redis
8
+
9
+ # TODO: Instead of just raising an exception, it would probably be better
10
+ # for the errback to set some sort of 'disconnected' state. Then
11
+ # whenever a deferrable was 'subscribed' we could instantly fail
12
+ # the deferrable with whatever connection error we had.
13
+ # An alternative which would have a similar result would be to
14
+ # subscribe lazily (i.e. not until we have a deferrable to subscribe).
15
+ # Then, if connecting failed, it'd be super easy to fail the deferrable
16
+ # with the same error.
17
+ # The final goal is to allow the failed deferrable bubble back up
18
+ # so we can send back a nice, clean 500 error to the client.
19
+ redis.subscribe('firehose:channel_updates').
20
+ errback{|e| EM.next_tick { raise e } }.
21
+ callback { Firehose.logger.debug "Redis subscribed to `firehose:channel_updates`" }
22
+ redis.on(:message) do |_, payload|
23
+ channel_key, sequence, message = Firehose::Publisher.from_payload(payload)
24
+
25
+ if deferrables = subscriptions.delete(channel_key)
26
+ Firehose.logger.debug "Redis notifying #{deferrables.count} deferrable(s) at `#{channel_key}` with sequence `#{sequence}` and message `#{message}`"
27
+ deferrables.each do |deferrable|
28
+ Firehose.logger.debug "Sending message #{message} and sequence #{sequence} to client from subscriber"
29
+ deferrable.succeed message, sequence.to_i
30
+ end
31
+ end
32
+ end
33
+ end
34
+
35
+ def subscribe(channel_key, deferrable)
36
+ subscriptions[channel_key].push deferrable
37
+ end
38
+
39
+ def unsubscribe(channel_key, deferrable)
40
+ subscriptions[channel_key].delete deferrable
41
+ subscriptions.delete(channel_key) if subscriptions[channel_key].empty?
42
+ end
43
+
44
+
45
+ private
46
+ def subscriptions
47
+ @subscriptions ||= Hash.new{|h,k| h[k] = []}
48
+ end
49
+
50
+ def key(*segments)
51
+ segments.unshift(:firehose).join(':')
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,23 @@
1
+ module Firehose
2
+ module SwfPolicyRequest
3
+
4
+ # Borrowed from: https://github.com/igrigorik/em-websocket/blob/3e7f7d7760cc23b9d1d34fc1c17bab4423b5d11a/lib/em-websocket/connection.rb#L104
5
+ def handle_swf_policy_request(data)
6
+ if data =~ /\A<policy-file-request\s*\/>/
7
+ Firehose.logger.debug "Received SWF Policy request: #{data.inspect}"
8
+ send_data policy
9
+ close_connection_after_writing
10
+ true
11
+ end
12
+ end
13
+
14
+ def policy
15
+ <<-EOS
16
+ <?xml version="1.0"?>
17
+ <cross-domain-policy>
18
+ <allow-access-from domain="*" to-ports="*"/>
19
+ </cross-domain-policy>
20
+ EOS
21
+ end
22
+ end
23
+ end
@@ -1,4 +1,4 @@
1
1
  module Firehose
2
- VERSION = "0.1.1"
3
- CODENAME = "Rails Ready"
2
+ VERSION = "0.2.alpha.2"
3
+ CODENAME = "Tricked-out Tools"
4
4
  end
@@ -0,0 +1,33 @@
1
+ # This file file monkeypatches Rainbows! to return a proper SWF policy file.
2
+ # Enable this with something like this in your config/rainbows.rb file:
3
+ #
4
+ # after_fork do |server, worker|
5
+ # require 'rainbows_em_swf_policy'
6
+ # end if ENV['RACK_ENV'] == 'development'
7
+ #
8
+ # You should only use this in development. It has not been well tested in a
9
+ # production environment.
10
+ #
11
+ # NOTE: This only works if you are using Rainbows! with EventMachine.
12
+ #
13
+ # Some helpful links:
14
+ # http://unicorn.bogomips.org/Unicorn/Configurator.html
15
+ # http://www.adobe.com/devnet/flashplayer/articles/socket_policy_files.html
16
+ # http://blog.vokle.com/index.php/2009/06/10/dealing-with-adobe-and-serving-socket-policy-servers-via-nginx-and-10-lines-of-code/
17
+
18
+ require 'rainbows'
19
+ # Ensure the class already exists so we are overwriting it.
20
+ Rainbows::EventMachine::Client
21
+
22
+ class Rainbows::EventMachine::Client
23
+ include Firehose::SwfPolicyRequest
24
+ alias_method :receive_data_without_swf_policy, :receive_data
25
+ # Borrowed from: https://github.com/igrigorik/em-websocket/blob/3e7f7d7760cc23b9d1d34fc1c17bab4423b5d11a/lib/em-websocket/connection.rb#L104
26
+ def receive_data(data)
27
+ if handle_swf_policy_request(data)
28
+ return false
29
+ else
30
+ receive_data_without_swf_policy(data)
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,26 @@
1
+ # This file file monkeypatches Thin to return a proper SWF policy file.
2
+ #
3
+ # You should only use this in development. It has not been well tested in a
4
+ # production environment.
5
+ #
6
+ # NOTE: This only works if you are using Thin with EventMachine.
7
+ #
8
+ # Some helpful links:
9
+ # http://www.adobe.com/devnet/flashplayer/articles/socket_policy_files.html
10
+ # http://blog.vokle.com/index.php/2009/06/10/dealing-with-adobe-and-serving-socket-policy-servers-via-nginx-and-10-lines-of-code/
11
+
12
+ require 'thin'
13
+ # Ensure the class already exists so we are overwriting it.
14
+ Thin::Connection
15
+
16
+ class Thin::Connection
17
+ include Firehose::SwfPolicyRequest
18
+ alias_method :receive_data_without_swf_policy, :receive_data
19
+ def receive_data(data)
20
+ if handle_swf_policy_request(data)
21
+ return false
22
+ else
23
+ receive_data_without_swf_policy(data)
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,16 @@
1
+ require 'spec_helper'
2
+
3
+ module IntegrationTestHelper
4
+ def start_server
5
+ @server_pid = fork do
6
+ Firehose::Server.new(:server => server, :port => uri.port).start
7
+ end
8
+
9
+ # Need to give the server a chance to boot up.
10
+ sleep 1
11
+ end
12
+
13
+ def stop_server
14
+ Process.kill 'INT', @server_pid
15
+ end
16
+ end
@@ -0,0 +1,7 @@
1
+ require 'spec_helper'
2
+ require 'integrations/shared_examples'
3
+
4
+ describe "rainbows" do
5
+ let(:server) { :rainbows }
6
+ it_behaves_like 'Firehose::Rack::App'
7
+ end
@@ -0,0 +1,111 @@
1
+ require 'spec_helper'
2
+ require 'integrations/integration_test_helper'
3
+
4
+ shared_examples_for 'Firehose::Rack::App' do
5
+ include EM::TestHelper
6
+ include IntegrationTestHelper
7
+
8
+ before(:all) do
9
+ Firehose::Producer.adapter = :em_http
10
+ start_server
11
+ end
12
+
13
+ after(:all) do
14
+ Firehose::Producer.adapter = nil
15
+ stop_server
16
+ end
17
+
18
+ before(:each) { WebMock.disable! }
19
+ after(:each) { WebMock.enable! }
20
+
21
+ let(:app) { Firehose::Rack::App.new }
22
+ let(:messages) { (1..2000).map{|n| "msg-#{n}" } }
23
+ let(:channel) { "/firehose/integration/#{Time.now.to_i}" }
24
+ let(:uri) { Firehose::Default::URI }
25
+ let(:http_url) { "http://#{uri.host}:#{uri.port}#{channel}" }
26
+ let(:ws_url) { "ws://#{uri.host}:#{uri.port}#{channel}" }
27
+
28
+ it "should pub-sub http and websockets" do
29
+ # Setup variables that we'll use after we turn off EM to validate our
30
+ # test assertions.
31
+ outgoing, received = messages.dup, Hash.new{|h,k| h[k] = []}
32
+
33
+ # Our WS and Http clients call this when they have received their messages to determine
34
+ # when to turn off EM and make the test assertion at the very bottom.
35
+ succeed = Proc.new do
36
+ # TODO: For some weird reason the `add_timer` call causes up to 20 seconds of delay after
37
+ # the test finishes running. However, without it the test will randomly fail with a
38
+ # "Redis disconnected" error.
39
+ em.add_timer(1) { em.stop } if received.values.all?{|arr| arr.size == messages.size }
40
+ end
41
+
42
+ # Setup a publisher
43
+ publish = Proc.new do
44
+ Firehose::Producer.new.publish(outgoing.shift).to(channel) do
45
+ # The random timer ensures that sometimes the clients will be behind
46
+ # and sometimes they will be caught up.
47
+ EM::add_timer(rand*0.005) { publish.call } unless outgoing.empty?
48
+ end
49
+ end
50
+
51
+ # Lets have an HTTP Long poll client
52
+ http_long_poll = Proc.new do |cid, last_sequence|
53
+ http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => last_sequence})
54
+ http.errback { em.stop }
55
+ http.callback do
56
+ received[cid] << http.response
57
+ if received[cid].size < messages.size
58
+ # Add some jitter so the clients aren't syncronized
59
+ EM::add_timer(rand*0.001) { http_long_poll.call cid, http.response_header['pragma'] }
60
+ else
61
+ succeed.call cid
62
+ end
63
+ end
64
+ end
65
+
66
+ # And test a web socket client too, at the same time.
67
+ websocket = Proc.new do |cid|
68
+ ws = Faye::WebSocket::Client.new(ws_url)
69
+ ws.onmessage = lambda do |event|
70
+ received[cid] << event.data
71
+ succeed.call cid unless received[cid].size < messages.size
72
+ end
73
+
74
+ ws.onclose = lambda do |event|
75
+ ws = nil
76
+ end
77
+
78
+ ws.onerror { |e| raise 'ws failed' + "\n" + e.inspect }
79
+ end
80
+
81
+ # Great, we have all the pieces in order, lets run this thing in the reactor.
82
+ em 60 do
83
+ # Start the clients.
84
+ websocket.call(1)
85
+ websocket.call(2)
86
+ http_long_poll.call(3)
87
+ http_long_poll.call(4)
88
+
89
+ # Wait a sec to let our clients set up.
90
+ em.add_timer(1){ publish.call }
91
+ end
92
+
93
+ # When EM stops, these assertions will be made.
94
+ received.size.should == 4
95
+ received.values.each do |arr|
96
+ arr.should == messages
97
+ end
98
+ end
99
+
100
+
101
+ it "should return 400 error for long-polling when using http long polling and sequence header is < 0" do
102
+ em 5 do
103
+ http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => -1})
104
+ http.errback { |e| raise e.inspect }
105
+ http.callback do
106
+ http.response_header.status.should == 400
107
+ em.stop
108
+ end
109
+ end
110
+ end
111
+ end
@@ -1,81 +1,7 @@
1
1
  require 'spec_helper'
2
+ require 'integrations/shared_examples'
2
3
 
3
- describe Firehose::Rack do
4
- include EM::TestHelper
5
-
6
- before(:all) do
7
- Firehose::Producer.adapter = :em_http
8
- end
9
-
10
- after(:all) do
11
- Firehose::Producer.adapter = nil
12
- end
13
-
14
- let(:app) { Firehose::Rack::App.new }
15
- let(:messages) { (1..2000).map(&:to_s) }
16
- let(:channel) { "/firehose/integration/#{Time.now.to_i}" }
17
- let(:uri) { Firehose::Default::URI }
18
- let(:http_url) { "http://#{uri.host}:#{uri.port}#{channel}" }
19
- let(:ws_url) { "ws://#{uri.host}:#{uri.port}#{channel}" }
20
-
21
- it "should pub-sub http and websockets" do
22
- # Setup variables that we'll use after we turn off EM to validate our
23
- # test assertions.
24
- outgoing, received_http, received_ws = messages.dup, [], []
25
-
26
- # Our WS and Http clients call this when they have received their messages to determine
27
- # when to turn off EM and make the test assertion at the very bottom.
28
- succeed = Proc.new do
29
- em.stop if received_http.size == messages.size and received_ws.size == messages.size
30
- end
31
-
32
- # Setup a publisher
33
- publish = Proc.new do
34
- Firehose::Producer.new.publish(outgoing.pop).to(channel) do
35
- publish.call unless outgoing.empty?
36
- end
37
- end
38
-
39
- # Lets have an HTTP Long poll client
40
- http_long_poll = Proc.new do
41
- http = EM::HttpRequest.new(http_url).get(:query => {'cid' => 'alpha'})
42
- http.errback { em.stop }
43
- http.callback do
44
- received_http << http.response
45
- if received_http.size < messages.size
46
- http_long_poll.call
47
- else
48
- succeed.call
49
- end
50
- end
51
- end
52
-
53
- # And test a web socket client too, at the same time.
54
- websocket = Proc.new do
55
- ws = EventMachine::WebSocketClient.connect("#{ws_url}?cid=bravo")
56
- ws.errback { em.stop }
57
- ws.stream do |msg|
58
- received_ws << msg
59
- succeed.call unless received_ws.size < messages.size
60
- end
61
- end
62
-
63
- # Great, we have all the pieces in order, lets run this thing in the reactor.
64
- em do
65
- # Start the server
66
- server = ::Thin::Server.new('0.0.0.0', uri.port, app)
67
- server.start
68
-
69
- # Start the http_long_pollr.
70
- http_long_poll.call
71
- websocket.call
72
-
73
- # Wait a sec to let our http_long_poll setup.
74
- em.add_timer(1){ publish.call }
75
- end
76
-
77
- # When EM stops, these assertions will be made.
78
- received_ws.should =~ messages
79
- received_http.should =~ messages
80
- end
81
- end
4
+ describe "thin" do
5
+ let(:server) { :thin }
6
+ it_behaves_like 'Firehose::Rack::App'
7
+ end