firehose 1.2.20 → 1.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +29 -0
  3. data/.dockerignore +2 -0
  4. data/.gitignore +3 -1
  5. data/.rubocop.yml +1156 -0
  6. data/.ruby-version +1 -0
  7. data/.travis.yml +3 -7
  8. data/CHANGELOG.md +15 -0
  9. data/Dockerfile +11 -0
  10. data/Gemfile +4 -2
  11. data/Procfile.dev +0 -1
  12. data/README.md +66 -8
  13. data/Rakefile +43 -32
  14. data/coffeelint.json +129 -0
  15. data/docker-compose.yml +17 -0
  16. data/firehose.gemspec +5 -9
  17. data/karma.config.coffee +89 -0
  18. data/lib/assets/javascripts/firehose.js.coffee +1 -2
  19. data/lib/assets/javascripts/firehose/consumer.js.coffee +18 -2
  20. data/lib/assets/javascripts/firehose/core.js.coffee +2 -1
  21. data/lib/assets/javascripts/firehose/long_poll.js.coffee +69 -8
  22. data/lib/assets/javascripts/firehose/multiplexed_consumer.js.coffee +74 -0
  23. data/lib/assets/javascripts/firehose/transport.js.coffee +4 -2
  24. data/lib/assets/javascripts/firehose/web_socket.js.coffee +51 -5
  25. data/lib/firehose/cli.rb +2 -1
  26. data/lib/firehose/client/producer.rb +10 -4
  27. data/lib/firehose/rack/consumer.rb +39 -0
  28. data/lib/firehose/rack/consumer/http_long_poll.rb +118 -45
  29. data/lib/firehose/rack/consumer/web_socket.rb +133 -28
  30. data/lib/firehose/rack/ping.rb +1 -1
  31. data/lib/firehose/rack/publisher.rb +10 -4
  32. data/lib/firehose/server.rb +9 -9
  33. data/lib/firehose/server/channel.rb +23 -31
  34. data/lib/firehose/server/message_buffer.rb +59 -0
  35. data/lib/firehose/server/publisher.rb +16 -17
  36. data/lib/firehose/server/redis.rb +32 -0
  37. data/lib/firehose/server/subscriber.rb +7 -7
  38. data/lib/firehose/version.rb +2 -2
  39. data/package.json +14 -2
  40. data/spec/integrations/shared_examples.rb +89 -7
  41. data/spec/javascripts/firehose/multiplexed_consumer_spec.coffee +72 -0
  42. data/spec/javascripts/firehose/transport_spec.coffee +0 -2
  43. data/spec/javascripts/firehose/websocket_spec.coffee +2 -0
  44. data/spec/javascripts/helpers/spec_helper.js +1 -0
  45. data/spec/javascripts/support/jquery-1.11.1.js +10308 -0
  46. data/{lib/assets/javascripts/vendor → spec/javascripts/support}/json2.js +0 -0
  47. data/spec/javascripts/support/spec_helper.coffee +3 -0
  48. data/spec/lib/assets_spec.rb +8 -8
  49. data/spec/lib/client/producer_spec.rb +14 -14
  50. data/spec/lib/firehose_spec.rb +2 -2
  51. data/spec/lib/rack/consumer/http_long_poll_spec.rb +21 -3
  52. data/spec/lib/rack/consumer_spec.rb +4 -4
  53. data/spec/lib/rack/ping_spec.rb +4 -4
  54. data/spec/lib/rack/publisher_spec.rb +5 -5
  55. data/spec/lib/server/app_spec.rb +2 -2
  56. data/spec/lib/server/channel_spec.rb +58 -44
  57. data/spec/lib/server/message_buffer_spec.rb +148 -0
  58. data/spec/lib/server/publisher_spec.rb +29 -22
  59. data/spec/lib/server/redis_spec.rb +13 -0
  60. data/spec/lib/server/subscriber_spec.rb +14 -13
  61. data/spec/spec_helper.rb +8 -1
  62. metadata +34 -95
  63. data/.rbenv-version +0 -1
  64. data/Guardfile +0 -31
  65. data/config/evergreen.rb +0 -9
@@ -10,6 +10,7 @@ module Firehose
10
10
  PublishError = Class.new(RuntimeError)
11
11
  TimeoutError = Class.new(Faraday::Error::TimeoutError)
12
12
  DEFAULT_TIMEOUT = 1 # How many seconds should we wait for a publish to take?
13
+ DEFAULT_ERROR_HANDLER = ->(e) { raise e }
13
14
 
14
15
  # A DSL for publishing requests. This doesn't so much, but lets us call
15
16
  # Firehose::Client::Producer::Http#publish('message').to('channel'). Slick eh? If you don't like it,
@@ -41,8 +42,9 @@ module Firehose
41
42
 
42
43
  # Publish the message via HTTP.
43
44
  def put(message, channel, opts, &block)
44
- ttl = opts[:ttl]
45
- timeout = opts[:timeout] || @timeout || DEFAULT_TIMEOUT
45
+ ttl = opts[:ttl]
46
+ timeout = opts[:timeout] || @timeout || DEFAULT_TIMEOUT
47
+ buffer_size = opts[:buffer_size]
46
48
 
47
49
  response = conn.put do |req|
48
50
  req.options[:timeout] = timeout
@@ -59,13 +61,17 @@ module Firehose
59
61
  end
60
62
  req.body = message
61
63
  req.headers['Cache-Control'] = "max-age=#{ttl.to_i}" if ttl
64
+ req.headers["X-Firehose-Buffer-Size"] = buffer_size.to_s if buffer_size
62
65
  end
63
66
  response.on_complete do
64
67
  case response.status
65
68
  when 202 # Fire off the callback if everything worked out OK.
66
69
  block.call(response) if block
67
70
  else
68
- error_handler.call PublishError.new("Could not publish #{message.inspect} to '#{uri.to_s}/#{channel}': #{response.inspect}")
71
+ # don't pass along basic auth header, if present
72
+ response_data = response.inspect.gsub(/"Authorization"=>"Basic \S+"/, '"Authorization" => "Basic [HIDDEN]"')
73
+ endpoint = "#{uri}/#{channel}".gsub(/:\/\/\S+@/, "://")
74
+ error_handler.call PublishError.new("Could not publish #{message.inspect} to '#{endpoint}': #{response_data}")
69
75
  end
70
76
  end
71
77
 
@@ -81,7 +87,7 @@ module Firehose
81
87
 
82
88
  # Raise an exception if an error occurs when connecting to the Firehose.
83
89
  def error_handler
84
- @error_handler || Proc.new{ |e| raise e }
90
+ @error_handler || DEFAULT_ERROR_HANDLER
85
91
  end
86
92
 
87
93
  # What adapter should Firehose use to PUT the message? List of adapters is
@@ -10,6 +10,45 @@ module Firehose
10
10
  autoload :HttpLongPoll, 'firehose/rack/consumer/http_long_poll'
11
11
  autoload :WebSocket, 'firehose/rack/consumer/web_socket'
12
12
 
13
+ MULTIPLEX_CHANNEL = "channels@firehose"
14
+
15
+ def self.multiplexing_request?(env)
16
+ env["PATH_INFO"].include? MULTIPLEX_CHANNEL
17
+ end
18
+
19
+ def self.multiplex_subscriptions(request)
20
+ if request.get?
21
+ query_string_subscriptions(request.env)
22
+ elsif request.post?
23
+ post_subscriptions(request)
24
+ end
25
+ end
26
+
27
+ def self.query_string_subscriptions(env)
28
+ query_params = ::Rack::Utils.parse_query(env["QUERY_STRING"])
29
+
30
+ query_params["subscribe"].to_s.split(",").map do |sub|
31
+ chan, last_sequence = sub.split("!")
32
+ last_sequence = last_sequence.to_i
33
+ last_sequence = 0 if last_sequence < 0
34
+ {
35
+ channel: chan,
36
+ message_sequence: last_sequence
37
+ }
38
+ end
39
+ end
40
+
41
+ def self.post_subscriptions(request)
42
+ body = request.body.read
43
+ subs = JSON.parse(body).map do |chan, last_sequence|
44
+ last_sequence = 0 if last_sequence < 0
45
+ {
46
+ channel: chan,
47
+ message_sequence: last_sequence
48
+ }
49
+ end
50
+ end
51
+
13
52
  # Let the client configure the consumer on initialization.
14
53
  def initialize
15
54
  yield self if block_given?
@@ -4,13 +4,10 @@ module Firehose
4
4
  module Rack
5
5
  class Consumer
6
6
  class HttpLongPoll
7
- include Firehose::Rack::Helpers
8
-
9
7
  # How long should we wait before closing out the consuming clients web connection
10
8
  # for long polling? Most browsers timeout after a connection has been idle for 30s.
11
9
  TIMEOUT = 20
12
10
 
13
- # Configures the timeout for the
14
11
  attr_accessor :timeout
15
12
 
16
13
  def initialize(timeout=TIMEOUT)
@@ -19,68 +16,144 @@ module Firehose
19
16
  end
20
17
 
21
18
  def call(env)
22
- req = env['parsed_request'] ||= ::Rack::Request.new(env)
23
- path = req.path
24
- method = req.request_method
25
- # Get the Last Message Sequence from the query string.
26
- # Ideally we'd use an HTTP header, but android devices don't let us
27
- # set any HTTP headers for CORS requests.
28
- last_sequence = req.params['last_message_sequence'].to_i
29
-
30
- case method
31
- # GET is how clients subscribe to the queue. When a messages comes in, we flush out a response,
32
- # close down the requeust, and the client then reconnects.
33
- when 'GET'
34
- Firehose.logger.debug "HTTP GET with last_sequence #{last_sequence} for path #{path} with query #{env["QUERY_STRING"].inspect} and params #{req.params.inspect}"
35
- EM.next_tick do
19
+ if Consumer.multiplexing_request?(env)
20
+ MultiplexingHandler.new(@timeout).call(env)
21
+ else
22
+ DefaultHandler.new(@timeout).call(env)
23
+ end
24
+ end
25
+
26
+ class Handler
27
+ include Firehose::Rack::Helpers
28
+
29
+ def initialize(timeout=TIMEOUT)
30
+ @timeout = timeout
31
+ yield self if block_given?
32
+ end
33
+
34
+ def call(env)
35
+ request = request(env)
36
+ method = request.request_method
37
+
38
+ case method
39
+ # GET is how clients subscribe to the queue. When a messages comes in, we flush out a response,
40
+ # close down the requeust, and the client then reconnects.
41
+ when "GET"
42
+ handle_request(request, env)
43
+ return ASYNC_RESPONSE
44
+ # we use post messages for http long poll multiplexing
45
+ when "POST"
46
+ if Consumer.multiplexing_request?(env)
47
+ handle_request(request, env)
48
+ return ASYNC_RESPONSE
49
+ end
50
+ end
51
+
52
+ Firehose.logger.debug "HTTP #{method} not supported"
53
+ response(405, "#{method} not supported.", "Allow" => "GET")
54
+ end
55
+
56
+ private
57
+
58
+ # If the request is a CORS request, return those headers, otherwise don't worry 'bout it
59
+ def response_headers(env)
60
+ cors_origin(env) ? cors_headers(env) : {}
61
+ end
62
+
63
+ def cors_origin(env)
64
+ env['HTTP_ORIGIN']
65
+ end
66
+
67
+ def cors_headers(env)
68
+ # TODO seperate out CORS logic as an async middleware with a Goliath web server.
69
+ {'Access-Control-Allow-Origin' => cors_origin(env)}
70
+ end
71
+
72
+ def request(env)
73
+ env['parsed_request'] ||= ::Rack::Request.new(env)
74
+ end
36
75
 
76
+ def async_callback(env, code, message = "", headers = nil)
77
+ resp_headers = response_headers(env)
78
+
79
+ if headers
80
+ resp_headers.merge!(headers)
81
+ end
82
+
83
+ if cb = env["async.callback"]
84
+ cb.call response(code, message, resp_headers)
85
+ else
86
+ Firehose.logger.error "async.callback not set for response: #{message.inspect}"
87
+ end
88
+ end
89
+
90
+ def respond_async(channel, last_sequence, env)
91
+ EM.next_tick do
37
92
  if last_sequence < 0
38
- env['async.callback'].call response(400, "The last_message_sequence parameter may not be less than zero", response_headers(env))
93
+ async_callback env, 400, "The last_message_sequence parameter may not be less than zero"
39
94
  else
40
- Server::Channel.new(path).next_message(last_sequence, :timeout => timeout).callback do |message, sequence|
41
- env['async.callback'].call response(200, wrap_frame(message, sequence), response_headers(env))
95
+ Server::Channel.new(channel).next_messages(last_sequence, :timeout => @timeout).callback do |messages|
96
+ # TODO: Can we send all of these messages down in one request? Sending one message per
97
+ # request is slow and inefficient. If we change the protocol (3.0?) we could batch the
98
+ # messages and send them all down the pipe, then close the conneciton.
99
+ message = messages.first
100
+ async_callback env, 200, wrap_frame(channel, message)
42
101
  end.errback do |e|
43
102
  if e == :timeout
44
- env['async.callback'].call response(204, '', response_headers(env))
103
+ async_callback env, 204
45
104
  else
46
- Firehose.logger.error "Unexpected error when trying to GET last_sequence #{last_sequence} for path #{path}: #{e.inspect}"
47
- env['async.callback'].call response(500, 'Unexpected error', response_headers(env))
105
+ Firehose.logger.error "Unexpected error when trying to GET last_sequence #{last_sequence} for path #{channel}: #{e.inspect}"
106
+ async_callback env, 500, "Unexpected error"
48
107
  end
49
108
  end
50
109
  end
51
-
52
110
  end
53
-
54
- # Tell the web server that this will be an async response.
55
- ASYNC_RESPONSE
56
-
57
- else
58
- Firehose.logger.debug "HTTP #{method} not supported"
59
- response(501, "#{method} not supported.")
60
111
  end
61
112
  end
62
113
 
114
+ class DefaultHandler < Handler
115
+ def wrap_frame(channel, message)
116
+ JSON.generate :message => message.payload, :last_sequence => message.sequence
117
+ end
63
118
 
64
- private
119
+ def log_request(path, last_sequence, env)
120
+ Firehose.logger.debug "HTTP GET with last_sequence #{last_sequence} for path #{path} with query #{env["QUERY_STRING"].inspect}"
121
+ end
65
122
 
66
- def wrap_frame(message, last_sequence)
67
- JSON.generate :message => message, :last_sequence => last_sequence
68
- end
123
+ def handle_request(request, env)
124
+ # Get the Last Message Sequence from the query string.
125
+ # Ideally we'd use an HTTP header, but android devices don't let us
126
+ # set any HTTP headers for CORS requests.
127
+ last_sequence = request.params['last_message_sequence'].to_i
128
+ channel = request.path
69
129
 
70
- # If the request is a CORS request, return those headers, otherwise don't worry 'bout it
71
- def response_headers(env)
72
- cors_origin(env) ? cors_headers(env) : {}
130
+ log_request channel, last_sequence, env
131
+ respond_async channel, last_sequence, env
132
+ end
73
133
  end
74
134
 
75
- def cors_origin(env)
76
- env['HTTP_ORIGIN']
77
- end
135
+ class MultiplexingHandler < Handler
136
+ def wrap_frame(channel, message)
137
+ JSON.generate channel: channel, :message => message.payload, :last_sequence => message.sequence
138
+ end
78
139
 
79
- def cors_headers(env)
80
- # TODO seperate out CORS logic as an async middleware with a Goliath web server.
81
- {'Access-Control-Allow-Origin' => cors_origin(env)}
140
+ def log_request(request, subscriptions, env)
141
+ if request.post?
142
+ Firehose.logger.debug "HTTP multiplexing POST, subscribing #{subscriptions.inspect}"
143
+ else
144
+ Firehose.logger.debug "HTTP multiplexing GET with query #{env["QUERY_STRING"].inspect}"
145
+ end
146
+ end
147
+
148
+ def handle_request(request, env)
149
+ subscriptions = Consumer.multiplex_subscriptions(request)
150
+ log_request request, subscriptions, env
151
+ subscriptions.each do |sub|
152
+ respond_async(sub[:channel], sub[:message_sequence], env)
153
+ end
154
+ end
82
155
  end
83
156
  end
84
157
  end
85
158
  end
86
- end
159
+ end
@@ -1,5 +1,6 @@
1
1
  require 'faye/websocket'
2
2
  require 'json'
3
+ require "rack/utils"
3
4
 
4
5
  module Firehose
5
6
  module Rack
@@ -8,7 +9,11 @@ module Firehose
8
9
  # Setup a handler for the websocket connection.
9
10
  def call(env)
10
11
  ws = Faye::WebSocket.new(env)
11
- Handler.new(ws)
12
+ if Consumer.multiplexing_request?(env)
13
+ MultiplexingHandler.new(ws)
14
+ else
15
+ DefaultHandler.new(ws)
16
+ end
12
17
  ws.rack_response
13
18
  end
14
19
 
@@ -17,9 +22,6 @@ module Firehose
17
22
  Faye::WebSocket.websocket?(env)
18
23
  end
19
24
 
20
- # Manages connection state for the web socket that's connected
21
- # by the Consumer::WebSocket class. Deals with message sequence,
22
- # connection, failures, and subscription state.
23
25
  class Handler
24
26
  def initialize(ws)
25
27
  @ws = ws
@@ -31,31 +33,36 @@ module Firehose
31
33
  @ws.onmessage = method :message
32
34
  end
33
35
 
34
- # Subscribe the client to the channel on the server. Asks for
35
- # the last sequence for clients that reconnect.
36
- def subscribe(last_sequence)
37
- @subscribed = true
38
- @channel = Server::Channel.new @req.path
39
- @deferrable = @channel.next_message last_sequence
40
- @deferrable.callback do |message, sequence|
41
- Firehose.logger.debug "WS sent `#{message}` to `#{@req.path}` with sequence `#{sequence}`"
42
- @ws.send self.class.wrap_frame(message, last_sequence)
43
- subscribe sequence
44
- end
45
- @deferrable.errback do |e|
46
- EM.next_tick { raise e.inspect } unless e == :disconnect
47
- end
36
+ def parse_message(event)
37
+ JSON.parse(event.data, :symbolize_names => true) rescue {}
38
+ end
39
+
40
+ # Send a JSON message to the client
41
+ # Expects message to be a Hash
42
+ def send_message(message)
43
+ @ws.send JSON.generate(message)
48
44
  end
49
45
 
46
+ # Log errors if a socket fails. `close` will fire after this to clean up any
47
+ # remaining connectons.
48
+ def error(event)
49
+ Firehose.logger.error "WS connection `#{@req.path}` error. Message: `#{event.message.inspect}`"
50
+ end
51
+ end
52
+
53
+ # Manages connection state for the web socket that's connected
54
+ # by the Consumer::WebSocket class. Deals with message sequence,
55
+ # connection, failures, and subscription state.
56
+ class DefaultHandler < Handler
50
57
  # Manages messages sent from the connect client to the server. This is mostly
51
58
  # used to handle heart-beats that are designed to prevent the WebSocket connection
52
59
  # from timing out from inactivity.
53
60
  def message(event)
54
- msg = JSON.parse(event.data, :symbolize_names => true) rescue {}
61
+ msg = parse_message(event)
55
62
  seq = msg[:message_sequence]
56
63
  if msg[:ping] == 'PING'
57
64
  Firehose.logger.debug "WS ping received, sending pong"
58
- @ws.send JSON.generate :pong => 'PONG'
65
+ send_message pong: "PONG"
59
66
  elsif !@subscribed && seq.kind_of?(Integer)
60
67
  Firehose.logger.debug "Subscribing at message_sequence #{seq}"
61
68
  subscribe seq
@@ -77,16 +84,114 @@ module Firehose
77
84
  Firehose.logger.debug "WS connection `#{@req.path}` closing. Code: #{event.code.inspect}; Reason #{event.reason.inspect}"
78
85
  end
79
86
 
80
- # Log errors if a socket fails. `close` will fire after this to clean up any
81
- # remaining connectons.
82
- def error(event)
83
- Firehose.logger.error "WS connection `#{@req.path}` error. Message: `#{event.message.inspect}`"
87
+ # Subscribe the client to the channel on the server. Asks for
88
+ # the last sequence for clients that reconnect.
89
+ def subscribe(last_sequence)
90
+ @subscribed = true
91
+ @channel = Server::Channel.new @req.path
92
+ @deferrable = @channel.next_messages last_sequence
93
+ @deferrable.callback do |messages|
94
+ messages.each do |message|
95
+ Firehose.logger.debug "WS sent `#{message.payload}` to `#{@req.path}` with sequence `#{message.sequence}`"
96
+ send_message message: message.payload, last_sequence: message.sequence
97
+ end
98
+ subscribe messages.last.sequence
99
+ end
100
+ @deferrable.errback do |e|
101
+ unless e == :disconnect
102
+ Firehose.logger.error "WS Error: #{e}"
103
+ EM.next_tick { raise e.inspect }
104
+ end
105
+ end
84
106
  end
107
+ end
85
108
 
86
- # Wrap a message in a sequence so that the client can record this and give us
87
- # the sequence when it reconnects.
88
- def self.wrap_frame(message, last_sequence)
89
- JSON.generate :message => message, :last_sequence => last_sequence
109
+ class MultiplexingHandler < Handler
110
+ class Subscription < Struct.new(:channel, :deferrable)
111
+ def close
112
+ deferrable.fail :disconnect
113
+ channel.unsubscribe(deferrable)
114
+ end
115
+ end
116
+
117
+ def initialize(ws)
118
+ super(ws)
119
+ @subscriptions = {}
120
+ subscribe_multiplexed Consumer.multiplex_subscriptions(@req)
121
+ end
122
+
123
+ def message(event)
124
+ msg = parse_message(event)
125
+
126
+ if subscriptions = msg[:multiplex_subscribe]
127
+ subscriptions = [subscriptions] unless subscriptions.is_a?(Array)
128
+ return subscribe_multiplexed(subscriptions)
129
+ end
130
+
131
+ if channel_names = msg[:multiplex_unsubscribe]
132
+ return unsubscribe(channel_names)
133
+ end
134
+
135
+ if msg[:ping] == 'PING'
136
+ Firehose.logger.debug "WS ping received, sending pong"
137
+ return send_message pong: "PONG"
138
+ end
139
+ end
140
+
141
+ def open(event)
142
+ Firehose.logger.debug "Multiplexing Websocket connected: #{@req.path}"
143
+ end
144
+
145
+ def close(event)
146
+ @subscriptions.each_value(&:close)
147
+ @subscriptions.clear
148
+ end
149
+
150
+ def subscribe_multiplexed(subscriptions)
151
+ subscriptions.each do |sub|
152
+ Firehose.logger.debug "Subscribing multiplexed to: #{sub}"
153
+
154
+ channel, sequence = sub[:channel], sub[:message_sequence]
155
+ next if channel.nil?
156
+
157
+ subscribe(channel, sequence.to_i)
158
+ end
159
+ end
160
+
161
+ # Subscribe the client to the channel on the server. Asks for
162
+ # the last sequence for clients that reconnect.
163
+ def subscribe(channel_name, last_sequence)
164
+ channel = Server::Channel.new channel_name
165
+ deferrable = channel.next_messages last_sequence
166
+ subscription = Subscription.new(channel, deferrable)
167
+
168
+ @subscriptions[channel_name] = subscription
169
+
170
+ deferrable.callback do |messages|
171
+ messages.each do |message|
172
+ send_message(
173
+ channel: channel_name,
174
+ message: message.payload,
175
+ last_sequence: message.sequence
176
+ )
177
+ Firehose.logger.debug "WS sent `#{message.payload}` to `#{channel_name}` with sequence `#{message.sequence}`"
178
+ end
179
+ subscribe channel_name, messages.last.sequence
180
+ end
181
+
182
+ deferrable.errback do |e|
183
+ EM.next_tick { raise e.inspect } unless e == :disconnect
184
+ end
185
+ end
186
+
187
+ def unsubscribe(channel_names)
188
+ Firehose.logger.debug "Unsubscribing from channels: #{channel_names}"
189
+ Array(channel_names).each do |chan|
190
+ if sub = @subscriptions[chan]
191
+ sub.close
192
+ @subscriptions.delete(chan)
193
+ end
194
+ end
90
195
  end
91
196
  end
92
197
  end