http_streaming_client 0.8.6 → 0.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG +8 -0
- data/README.md +46 -31
- data/lib/http_streaming_client/client.rb +196 -147
- data/lib/http_streaming_client/decoders/gzip.rb +1 -1
- data/lib/http_streaming_client/errors.rb +3 -0
- data/lib/http_streaming_client/version.rb +1 -1
- data/spec/reconnect_spec.rb +32 -0
- data/spec/spec_helper.rb +2 -0
- data/tools/adobe_firehose_performance_test.rb +4 -2
- data/tools/twitter_firehose_performance_test.rb +2 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8308d062ebb595dddc19e1ce5232829fb92bd9ce
|
4
|
+
data.tar.gz: cfbe2c60d0b7b021cbe09d80bee4ef8114115039
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 50ee772c4e19aec32f3a771b1686b8beadd9d7fb2038cf08b0b65792f7b070f26b1bf74519c4f85deea9462e2a93df0939888d3c1233a383a6b21286ce93a6c1
|
7
|
+
data.tar.gz: 323dec68b2c4d8730d4c9803358f142e721efc3aa2b08745625eacd4ea6b59351e2394a24ca739d5baa5ef64f1343f2060d538a7d3bc2c7f3b3fdded0f17d066
|
data/CHANGELOG
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
http_streaming_client 0.8.8 (1.7.2014)
|
2
|
+
========================================
|
3
|
+
* Fixed Issue #1, memory leak due to accumulated messages within the buffer pipeline
|
4
|
+
* Added automatic reconnection support and configuration options
|
5
|
+
* Added automatic HTTP 301 redirect support
|
6
|
+
* Added new unit tests for reconnect and for memory leak detection
|
7
|
+
* Updated performance tests
|
8
|
+
|
1
9
|
http_streaming_client 0.8.5 (12.10.2013)
|
2
10
|
========================================
|
3
11
|
* Updates to support MRI ruby 2.0.0-p353
|
data/README.md
CHANGED
@@ -9,39 +9,69 @@ Ruby HTTP client with support for HTTP 1.1 streaming, GZIP and zlib compressed s
|
|
9
9
|
|
10
10
|
MRI ruby-2.0.0-p353. If you need it, install via rvm: https://rvm.io/
|
11
11
|
|
12
|
-
## Installation
|
12
|
+
## Installation
|
13
13
|
|
14
|
-
|
14
|
+
Add this line to your application's Gemfile:
|
15
15
|
|
16
|
-
|
16
|
+
gem 'http_streaming_client'
|
17
17
|
|
18
|
-
|
18
|
+
And then execute:
|
19
|
+
|
20
|
+
$ bundle install
|
21
|
+
|
22
|
+
Or install it directly via gem with:
|
19
23
|
|
20
24
|
$ gem install http_streaming_client
|
21
25
|
|
22
|
-
##
|
26
|
+
## Simple Example
|
23
27
|
|
24
|
-
|
28
|
+
Twitter Firehose Sample Stream:
|
25
29
|
|
26
|
-
|
30
|
+
```ruby
|
31
|
+
require 'http_streaming_client'
|
27
32
|
|
28
|
-
|
33
|
+
twitter_stream_url = "https://stream.twitter.com/1.1/statuses/sample.json"
|
29
34
|
|
30
|
-
|
35
|
+
# Generate the HMAC-SHA1 Twitter OAuth authorization header
|
36
|
+
authorization = HttpStreamingClient::Oauth::Twitter.generate_authorization(twitter_stream_url, "get", {}, OAUTH_CONSUMER_KEY, OAUTH_CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
|
31
37
|
|
32
|
-
|
38
|
+
# Configure the client
|
39
|
+
client = HttpStreamingClient::Client.new(compression: true, reconnect: true, reconnect_interval: 10, reconnect_attempts: 60)
|
33
40
|
|
34
|
-
|
41
|
+
# Open the connection and start processing messages
|
42
|
+
response = client.get(twitter_stream_url, {:headers => {'Authorization' => "#{authorization}" }}) { |line|
|
43
|
+
logger.info "Received a line that we could parse into JSON if we want: #{line}"
|
44
|
+
client.interrupt if we_want_to_stop_receiving_messages?
|
45
|
+
}
|
46
|
+
```
|
35
47
|
|
36
|
-
|
48
|
+
For more examples, take a look at
|
37
49
|
|
38
|
-
|
50
|
+
* spec/client_spec.rb
|
51
|
+
* tools/adobe_firehose.rb
|
52
|
+
* tools/adobe_firehose_performance_test.rb
|
53
|
+
* tools/twitter_firehose.rb
|
54
|
+
* tools/twitter_firehose_performance_test.rb
|
39
55
|
|
40
|
-
|
56
|
+
## Client Configuration Options
|
41
57
|
|
42
|
-
|
58
|
+
The following options are supported as hash option parameters for HttpStreamingClient::Client.new:
|
43
59
|
|
44
|
-
|
60
|
+
GZip Compression
|
61
|
+
|
62
|
+
compression: true/false (default: true)
|
63
|
+
|
64
|
+
Automatic Socket Reconnect Functions
|
65
|
+
|
66
|
+
reconnect: true/false (default: false)
|
67
|
+
|
68
|
+
Reconnect Interval
|
69
|
+
|
70
|
+
reconnect_interval: interval_seconds (default: 1 second)
|
71
|
+
|
72
|
+
Maximum Reconnect Attempts
|
73
|
+
|
74
|
+
reconnect_attempts: num_attempts (default: 10)
|
45
75
|
|
46
76
|
## Logging
|
47
77
|
|
@@ -110,21 +140,6 @@ Individual test suites in the spec directory can be run via:
|
|
110
140
|
|
111
141
|
An HTML coverage report is generated at the end of a full test run in the coverage directory.
|
112
142
|
|
113
|
-
## Examples
|
114
|
-
|
115
|
-
Take a look at
|
116
|
-
|
117
|
-
* spec/client_spec.rb
|
118
|
-
* tools/adobe_firehose.rb
|
119
|
-
* tools/adobe_firehose_performance_test.rb
|
120
|
-
* tools/twitter_firehose.rb
|
121
|
-
* tools/twitter_firehose_performance_test.rb
|
122
|
-
|
123
|
-
## TODO
|
124
|
-
|
125
|
-
* connection management with reconnect functions
|
126
|
-
* load/memory testing
|
127
|
-
|
128
143
|
## Fixed Issues
|
129
144
|
|
130
145
|
* See [](CHANGELOG)
|
@@ -40,10 +40,13 @@ module HttpStreamingClient
|
|
40
40
|
|
41
41
|
class Client
|
42
42
|
|
43
|
-
attr_accessor :socket, :interrupted, :compression_requested
|
43
|
+
attr_accessor :socket, :interrupted, :compression_requested, :reconnect_requested, :reconnect_attempts, :reconnect_interval
|
44
44
|
|
45
45
|
ALLOWED_MIME_TYPES = ["application/json", "text/plain", "text/html"]
|
46
46
|
|
47
|
+
MAX_RECONNECT_ATTEMPTS = 10
|
48
|
+
RECONNECT_INTERVAL_SEC = 1
|
49
|
+
|
47
50
|
def self.logger
|
48
51
|
HttpStreamingClient.logger
|
49
52
|
end
|
@@ -56,8 +59,24 @@ module HttpStreamingClient
|
|
56
59
|
logger.debug("Client.new: #{opts}")
|
57
60
|
@socket = nil
|
58
61
|
@interrupted = false
|
62
|
+
|
59
63
|
@compression_requested = opts[:compression].nil? ? true : opts[:compression]
|
60
64
|
logger.debug("compression is #{@compression_requested}")
|
65
|
+
|
66
|
+
@reconnect_requested = opts[:reconnect].nil? ? false : opts[:reconnect]
|
67
|
+
logger.debug("reconnect is #{@reconnect_requested}")
|
68
|
+
|
69
|
+
if @reconnect_requested then
|
70
|
+
@reconnect_count = 0
|
71
|
+
|
72
|
+
@reconnect_attempts = opts[:reconnect_attempts].nil? ? MAX_RECONNECT_ATTEMPTS : opts[:reconnect_attempts]
|
73
|
+
logger.debug("reconnect_attempts is #{@reconnect_attempts}")
|
74
|
+
|
75
|
+
@reconnect_interval = opts[:reconnect_interval].nil? ? RECONNECT_INTERVAL_SEC : opts[:reconnect_interval]
|
76
|
+
logger.debug("reconnect_interval is #{@reconnect_interval}")
|
77
|
+
|
78
|
+
end
|
79
|
+
|
61
80
|
end
|
62
81
|
|
63
82
|
def self.get(uri, opts = {}, &block)
|
@@ -132,199 +151,226 @@ module HttpStreamingClient
|
|
132
151
|
headers = default_headers.merge(opts[:headers] || {})
|
133
152
|
logger.debug "request headers: #{headers}"
|
134
153
|
|
135
|
-
|
136
|
-
request = "#{method} #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
|
137
|
-
request << "Host: #{uri.host}\r\n"
|
138
|
-
headers.each do |k, v|
|
139
|
-
request << "#{k}: #{v}\r\n"
|
140
|
-
end
|
141
|
-
request << "\r\n"
|
142
|
-
if method == "POST"
|
143
|
-
request << body
|
144
|
-
end
|
154
|
+
begin
|
145
155
|
|
146
|
-
|
156
|
+
socket = initialize_socket(uri, opts)
|
147
157
|
|
148
|
-
|
149
|
-
response_head[:headers] = {}
|
158
|
+
@reconnect_count = 0 if @reconnect_requested
|
150
159
|
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
header = line.split(": ")
|
156
|
-
if header.size == 1
|
157
|
-
header = header[0].split(" ")
|
158
|
-
response_head[:version] = header[0]
|
159
|
-
response_head[:code] = header[1].to_i
|
160
|
-
response_head[:msg] = header[2]
|
161
|
-
logger.debug "HTTP response code is #{response_head[:code]}"
|
162
|
-
else
|
163
|
-
response_head[:headers][camelize_header_name(header[0])] = header[1].strip
|
164
|
-
end
|
160
|
+
request = "#{method} #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
|
161
|
+
request << "Host: #{uri.host}\r\n"
|
162
|
+
headers.each do |k, v|
|
163
|
+
request << "#{k}: #{v}\r\n"
|
165
164
|
end
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
if response_head[:code] == 301 then
|
171
|
-
location = response_head[:headers]["Location"]
|
172
|
-
raise InvalidRedirect, "Unable to find Location header for HTTP 301 response" if location.nil?
|
173
|
-
logger.debug "Received HTTP 301 redirect to #{location}, following..."
|
174
|
-
socket.close if !socket.nil? and !socket.closed?
|
175
|
-
opts.delete(:socket)
|
176
|
-
return request(method, location, opts, &block)
|
177
|
-
end
|
178
|
-
|
179
|
-
content_length = response_head[:headers]["Content-Length"].to_i
|
180
|
-
logger.debug "content-length: #{content_length}"
|
181
|
-
|
182
|
-
content_type = response_head[:headers]["Content-Type"].split(';').first
|
183
|
-
logger.debug "content-type: #{content_type}"
|
184
|
-
|
185
|
-
response_compression = false
|
186
|
-
|
187
|
-
if ALLOWED_MIME_TYPES.include?(content_type)
|
188
|
-
case response_head[:headers]["Content-Encoding"]
|
189
|
-
when "gzip"
|
190
|
-
response_compression = true
|
165
|
+
request << "\r\n"
|
166
|
+
if method == "POST"
|
167
|
+
request << body
|
191
168
|
end
|
192
|
-
else
|
193
|
-
raise InvalidContentType, "invalid response MIME type: #{content_type}"
|
194
|
-
end
|
195
169
|
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
logger.debug "response: #{response}"
|
201
|
-
raise HttpError.new(response_head[:code], "Received HTTP #{response_head[:code]} response", response_head[:headers])
|
202
|
-
end
|
170
|
+
socket.write(request)
|
171
|
+
|
172
|
+
response_head = {}
|
173
|
+
response_head[:headers] = {}
|
203
174
|
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
response = ""
|
208
|
-
|
209
|
-
if response_compression then
|
210
|
-
logger.debug "response compression detected"
|
211
|
-
if block_given? then
|
212
|
-
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
213
|
-
logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
|
214
|
-
block.call(line) unless @interrupted }
|
175
|
+
socket.each_line do |line|
|
176
|
+
if line == "\r\n" then
|
177
|
+
break
|
215
178
|
else
|
216
|
-
|
217
|
-
|
218
|
-
|
179
|
+
header = line.split(": ")
|
180
|
+
if header.size == 1
|
181
|
+
header = header[0].split(" ")
|
182
|
+
response_head[:version] = header[0]
|
183
|
+
response_head[:code] = header[1].to_i
|
184
|
+
response_head[:msg] = header[2]
|
185
|
+
logger.debug "HTTP response code is #{response_head[:code]}"
|
186
|
+
else
|
187
|
+
response_head[:headers][camelize_header_name(header[0])] = header[1].strip
|
188
|
+
end
|
219
189
|
end
|
220
190
|
end
|
221
191
|
|
222
|
-
|
223
|
-
chunkLeft = 0
|
192
|
+
logger.debug "response headers:#{response_head[:headers]}"
|
224
193
|
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
194
|
+
if response_head[:code] == 301 then
|
195
|
+
location = response_head[:headers]["Location"]
|
196
|
+
raise InvalidRedirect, "Unable to find Location header for HTTP 301 response" if location.nil?
|
197
|
+
logger.debug "Received HTTP 301 redirect to #{location}, following..."
|
198
|
+
socket.close if !socket.nil? and !socket.closed?
|
199
|
+
opts.delete(:socket)
|
200
|
+
return request(method, location, opts, &block)
|
201
|
+
end
|
230
202
|
|
231
|
-
|
203
|
+
content_length = response_head[:headers]["Content-Length"].to_i
|
204
|
+
logger.debug "content-length: #{content_length}"
|
232
205
|
|
233
|
-
|
234
|
-
|
206
|
+
content_type = response_head[:headers]["Content-Type"].split(';').first
|
207
|
+
logger.debug "content-type: #{content_type}"
|
235
208
|
|
236
|
-
|
237
|
-
next if partial.nil?
|
209
|
+
response_compression = false
|
238
210
|
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
remaining = size-partial.size
|
244
|
-
logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
|
211
|
+
if ALLOWED_MIME_TYPES.include?(content_type)
|
212
|
+
case response_head[:headers]["Content-Encoding"]
|
213
|
+
when "gzip"
|
214
|
+
response_compression = true
|
245
215
|
end
|
216
|
+
else
|
217
|
+
raise InvalidContentType, "invalid response MIME type: #{content_type}"
|
218
|
+
end
|
219
|
+
|
220
|
+
if (response_head[:code] != 200)
|
221
|
+
s = "Received HTTP #{response_head[:code]} response"
|
222
|
+
logger.debug "request: #{request}"
|
223
|
+
response = socket.read(content_length)
|
224
|
+
logger.debug "response: #{response}"
|
225
|
+
raise HttpError.new(response_head[:code], "Received HTTP #{response_head[:code]} response", response_head[:headers])
|
226
|
+
end
|
246
227
|
|
247
|
-
|
228
|
+
if response_head[:headers]["Transfer-Encoding"] == 'chunked'
|
229
|
+
partial = nil
|
230
|
+
decoder = nil
|
231
|
+
response = ""
|
248
232
|
|
249
233
|
if response_compression then
|
250
|
-
|
251
|
-
else
|
234
|
+
logger.debug "response compression detected"
|
252
235
|
if block_given? then
|
253
|
-
|
236
|
+
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
237
|
+
logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
|
238
|
+
block.call(line) unless @interrupted }
|
254
239
|
else
|
255
|
-
|
256
|
-
|
240
|
+
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
241
|
+
logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
|
242
|
+
response << line unless @interrupted }
|
257
243
|
end
|
258
244
|
end
|
259
|
-
end
|
260
245
|
|
261
|
-
|
262
|
-
|
263
|
-
else
|
264
|
-
# Not chunked transfer encoding, but potentially gzip'd, and potentially streaming with content-length = 0
|
265
|
-
|
266
|
-
if content_length > 0 then
|
267
|
-
bits = socket.read(content_length)
|
268
|
-
logger.debug "read #{content_length} bytes"
|
269
|
-
return bits if !response_compression
|
270
|
-
logger.debug "response compression detected"
|
271
|
-
begin
|
272
|
-
decoder = Zlib::GzipReader.new(StringIO.new(bits))
|
273
|
-
return decoder.read
|
274
|
-
rescue Zlib::Error
|
275
|
-
raise DecoderError
|
276
|
-
end
|
277
|
-
end
|
246
|
+
while !socket.eof? && (line = socket.gets)
|
247
|
+
chunkLeft = 0
|
278
248
|
|
279
|
-
|
249
|
+
if line.match /^0\r\n/ then
|
250
|
+
logger.debug "received zero length chunk, chunked encoding EOF"
|
251
|
+
logger.debug "EOF line: #{line}"
|
252
|
+
break
|
253
|
+
end
|
280
254
|
|
281
|
-
|
282
|
-
decoder = nil
|
283
|
-
response = ""
|
255
|
+
next if line == "\r\n"
|
284
256
|
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
257
|
+
size = line.hex
|
258
|
+
logger.debug "chunk size:#{size}"
|
259
|
+
|
260
|
+
partial = socket.read(size)
|
261
|
+
next if partial.nil?
|
262
|
+
|
263
|
+
remaining = size-partial.size
|
264
|
+
logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
|
265
|
+
until remaining == 0
|
266
|
+
partial << socket.read(remaining)
|
267
|
+
remaining = size-partial.size
|
268
|
+
logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
|
269
|
+
end
|
270
|
+
|
271
|
+
return if @interrupted
|
294
272
|
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
273
|
+
if response_compression then
|
274
|
+
decoder << partial
|
275
|
+
else
|
276
|
+
if block_given? then
|
277
|
+
yield partial
|
278
|
+
else
|
279
|
+
logger.debug "no block specified, returning chunk results and halting streaming response"
|
280
|
+
response << partial
|
281
|
+
interrupt
|
282
|
+
end
|
283
|
+
end
|
299
284
|
end
|
300
|
-
logger.debug "EOF detected"
|
301
|
-
decoder = nil
|
302
285
|
|
286
|
+
logger.debug "socket EOF detected" if socket.eof?
|
287
|
+
raise ReconnectRequest if @reconnect_requested
|
303
288
|
return response
|
304
289
|
|
305
290
|
else
|
291
|
+
# Not chunked transfer encoding, but potentially gzip'd, and potentially streaming with content-length = 0
|
292
|
+
|
293
|
+
if content_length > 0 then
|
294
|
+
bits = socket.read(content_length)
|
295
|
+
logger.debug "read #{content_length} bytes"
|
296
|
+
return bits if !response_compression
|
297
|
+
logger.debug "response compression detected"
|
298
|
+
begin
|
299
|
+
decoder = Zlib::GzipReader.new(StringIO.new(bits))
|
300
|
+
return decoder.read
|
301
|
+
rescue Zlib::Error
|
302
|
+
raise DecoderError
|
303
|
+
end
|
304
|
+
end
|
306
305
|
|
307
|
-
|
306
|
+
if response_compression then
|
307
|
+
|
308
|
+
logger.debug "response compression detected"
|
309
|
+
decoder = nil
|
310
|
+
response = ""
|
308
311
|
|
309
|
-
while (!socket.eof? and !(line = socket.readline).nil?)
|
310
312
|
if block_given? then
|
311
|
-
|
312
|
-
|
313
|
+
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
314
|
+
logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
|
315
|
+
block.call(line) unless @interrupted }
|
313
316
|
else
|
314
|
-
|
315
|
-
|
317
|
+
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
318
|
+
logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
|
319
|
+
response << line unless @interrupted }
|
316
320
|
end
|
317
|
-
break if @interrupted
|
318
|
-
end
|
319
321
|
|
320
|
-
|
322
|
+
while (!socket.eof? and !(line = socket.read_nonblock(2048)).nil?)
|
323
|
+
logger.debug "read compressed line, #{line.size} bytes"
|
324
|
+
decoder << line
|
325
|
+
break response if @interrupted
|
326
|
+
end
|
327
|
+
|
328
|
+
logger.debug "EOF detected"
|
329
|
+
raise ReconnectRequest if @reconnect_requested
|
330
|
+
return response
|
331
|
+
|
332
|
+
else
|
333
|
+
|
334
|
+
response = ""
|
335
|
+
|
336
|
+
while (!socket.eof? and !(line = socket.readline).nil?)
|
337
|
+
if block_given? then
|
338
|
+
yield line
|
339
|
+
logger.debug "read #{line.size} bytes"
|
340
|
+
else
|
341
|
+
logger.debug "read #{line.size} bytes, #{response.size} bytes total"
|
342
|
+
response << line
|
343
|
+
end
|
344
|
+
break if @interrupted
|
345
|
+
end
|
346
|
+
|
347
|
+
raise ReconnectRequest if @reconnect_requested
|
348
|
+
return response
|
349
|
+
|
350
|
+
end
|
351
|
+
end
|
352
|
+
rescue => e
|
353
|
+
return if @interrupted
|
354
|
+
logger.debug "Error Detected: #{e}" unless e.instance_of? ReconnectRequest
|
355
|
+
decoder.close if !decoder.nil?
|
356
|
+
socket.close if !socket.nil? and !socket.closed?
|
357
|
+
opts.delete(:socket)
|
321
358
|
|
359
|
+
if @reconnect_requested then
|
360
|
+
logger.info "Connection closed. Reconnect requested. Trying..."
|
361
|
+
@reconnect_count = @reconnect_count + 1
|
362
|
+
sleep @reconnect_interval
|
363
|
+
retry if @reconnect_count < @reconnect_attempts
|
364
|
+
logger.info "Maximum number of failed reconnect attempts reached (#{@reconnect_attempts}). Exiting."
|
322
365
|
end
|
366
|
+
|
367
|
+
raise e unless e.instance_of? ReconnectRequest
|
323
368
|
end
|
324
369
|
ensure
|
325
370
|
logger.debug "ensure socket closed"
|
326
371
|
decoder.close if !decoder.nil?
|
327
372
|
socket.close if !socket.nil? and !socket.closed?
|
373
|
+
opts.delete(:socket)
|
328
374
|
end
|
329
375
|
|
330
376
|
private
|
@@ -340,6 +386,8 @@ module HttpStreamingClient
|
|
340
386
|
uri = URI.parse(uri)
|
341
387
|
end
|
342
388
|
|
389
|
+
logger.debug "Connecting to #{uri.host}:#{uri.port}..."
|
390
|
+
|
343
391
|
@socket = TCPSocket.new(uri.host, uri.port)
|
344
392
|
|
345
393
|
if uri.scheme.eql? "https"
|
@@ -351,6 +399,7 @@ module HttpStreamingClient
|
|
351
399
|
end
|
352
400
|
end
|
353
401
|
|
402
|
+
logger.debug "Connected to #{uri.host}:#{uri.port}"
|
354
403
|
opts.merge!({:socket => @socket})
|
355
404
|
@interrupted = false
|
356
405
|
return opts[:socket]
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
|
4
|
+
describe HttpStreamingClient do
|
5
|
+
|
6
|
+
# currently disabled, requires a server that can be killed to simulate dropped connections
|
7
|
+
|
8
|
+
it "should receive exactly 10 messages, no reconnect" do
|
9
|
+
#it "should receive exactly 10 messages, no reconnect", :disabled => true do
|
10
|
+
|
11
|
+
count = 0
|
12
|
+
client = HttpStreamingClient::Client.new(compression: false)
|
13
|
+
response = client.get("http://localhost:3000/outbounds/consumer/1") { |line|
|
14
|
+
logger.debug "line received: #{line}"
|
15
|
+
count = count + 1
|
16
|
+
}
|
17
|
+
expect(response).to be_true
|
18
|
+
expect(count).to be(10)
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should reconnect on any error or EOF" do
|
22
|
+
|
23
|
+
client = HttpStreamingClient::Client.new(compression: false, reconnect: true, reconnect_attempts: 5, reconnect_interval: 1)
|
24
|
+
count = 0
|
25
|
+
response = client.get("http://localhost:3000/outbounds/consumer/1") { |line|
|
26
|
+
logger.debug "line received: #{line}"
|
27
|
+
count = count + 1
|
28
|
+
client.interrupt if count > 20
|
29
|
+
}
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -15,8 +15,8 @@ authorization = HttpStreamingClient::Oauth::Adobe.generate_authorization(url, US
|
|
15
15
|
puts "#{TOKENAPIHOST}:access token: #{authorization}"
|
16
16
|
client = HttpStreamingClient::Client.new(compression: false)
|
17
17
|
|
18
|
-
NUM_RECORDS_PER_BATCH =
|
19
|
-
MAX_RECORDS =
|
18
|
+
NUM_RECORDS_PER_BATCH = 5000
|
19
|
+
MAX_RECORDS = 3600000
|
20
20
|
|
21
21
|
count = 0
|
22
22
|
totalSize = 0
|
@@ -25,6 +25,7 @@ startTime = nil
|
|
25
25
|
lastTime = nil
|
26
26
|
|
27
27
|
puts "starting performance test run: #{Time.new.to_s}"
|
28
|
+
puts "stream: #{STREAMURL}"
|
28
29
|
|
29
30
|
startTime = lastTime = Time.new
|
30
31
|
|
@@ -69,6 +70,7 @@ response = client.get(STREAMURL, {:headers => {'Authorization' => "Bearer #{auth
|
|
69
70
|
stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
|
70
71
|
|
71
72
|
puts stats.to_json
|
73
|
+
STDOUT.flush
|
72
74
|
|
73
75
|
lastTime = now
|
74
76
|
intervalSize = 0
|
@@ -24,6 +24,7 @@ startTime = nil
|
|
24
24
|
lastTime = nil
|
25
25
|
|
26
26
|
puts "starting performance test run: #{Time.new.to_s}"
|
27
|
+
puts "stream: #{url}"
|
27
28
|
|
28
29
|
startTime = lastTime = Time.new
|
29
30
|
|
@@ -64,6 +65,7 @@ response = client.get(url, {:headers => {'Authorization' => "#{authorization}" }
|
|
64
65
|
stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
|
65
66
|
|
66
67
|
puts stats.to_json
|
68
|
+
STDOUT.flush
|
67
69
|
|
68
70
|
lastTime = now
|
69
71
|
intervalSize = 0
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: http_streaming_client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Tompkins
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-01-
|
11
|
+
date: 2014-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -128,6 +128,7 @@ files:
|
|
128
128
|
- spec/adobe_spec.rb
|
129
129
|
- spec/client_spec.rb
|
130
130
|
- spec/oauth_spec.rb
|
131
|
+
- spec/reconnect_spec.rb
|
131
132
|
- spec/spec_helper.rb
|
132
133
|
- spec/twitter_spec.rb
|
133
134
|
- tools/adobe_firehose.rb
|
@@ -164,5 +165,6 @@ test_files:
|
|
164
165
|
- spec/adobe_spec.rb
|
165
166
|
- spec/client_spec.rb
|
166
167
|
- spec/oauth_spec.rb
|
168
|
+
- spec/reconnect_spec.rb
|
167
169
|
- spec/spec_helper.rb
|
168
170
|
- spec/twitter_spec.rb
|