http_streaming_client 0.8.6 → 0.8.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG +8 -0
- data/README.md +46 -31
- data/lib/http_streaming_client/client.rb +196 -147
- data/lib/http_streaming_client/decoders/gzip.rb +1 -1
- data/lib/http_streaming_client/errors.rb +3 -0
- data/lib/http_streaming_client/version.rb +1 -1
- data/spec/reconnect_spec.rb +32 -0
- data/spec/spec_helper.rb +2 -0
- data/tools/adobe_firehose_performance_test.rb +4 -2
- data/tools/twitter_firehose_performance_test.rb +2 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8308d062ebb595dddc19e1ce5232829fb92bd9ce
|
4
|
+
data.tar.gz: cfbe2c60d0b7b021cbe09d80bee4ef8114115039
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 50ee772c4e19aec32f3a771b1686b8beadd9d7fb2038cf08b0b65792f7b070f26b1bf74519c4f85deea9462e2a93df0939888d3c1233a383a6b21286ce93a6c1
|
7
|
+
data.tar.gz: 323dec68b2c4d8730d4c9803358f142e721efc3aa2b08745625eacd4ea6b59351e2394a24ca739d5baa5ef64f1343f2060d538a7d3bc2c7f3b3fdded0f17d066
|
data/CHANGELOG
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
http_streaming_client 0.8.8 (1.7.2014)
|
2
|
+
========================================
|
3
|
+
* Fixed Issue #1, memory leak due to accumulated messages within the buffer pipeline
|
4
|
+
* Added automatic reconnection support and configuration options
|
5
|
+
* Added automatic HTTP 301 redirect support
|
6
|
+
* Added new unit tests for reconnect and for memory leak detection
|
7
|
+
* Updated performance tests
|
8
|
+
|
1
9
|
http_streaming_client 0.8.5 (12.10.2013)
|
2
10
|
========================================
|
3
11
|
* Updates to support MRI ruby 2.0.0-p353
|
data/README.md
CHANGED
@@ -9,39 +9,69 @@ Ruby HTTP client with support for HTTP 1.1 streaming, GZIP and zlib compressed s
|
|
9
9
|
|
10
10
|
MRI ruby-2.0.0-p353. If you need it, install via rvm: https://rvm.io/
|
11
11
|
|
12
|
-
## Installation
|
12
|
+
## Installation
|
13
13
|
|
14
|
-
|
14
|
+
Add this line to your application's Gemfile:
|
15
15
|
|
16
|
-
|
16
|
+
gem 'http_streaming_client'
|
17
17
|
|
18
|
-
|
18
|
+
And then execute:
|
19
|
+
|
20
|
+
$ bundle install
|
21
|
+
|
22
|
+
Or install it directly via gem with:
|
19
23
|
|
20
24
|
$ gem install http_streaming_client
|
21
25
|
|
22
|
-
##
|
26
|
+
## Simple Example
|
23
27
|
|
24
|
-
|
28
|
+
Twitter Firehose Sample Stream:
|
25
29
|
|
26
|
-
|
30
|
+
```ruby
|
31
|
+
require 'http_streaming_client'
|
27
32
|
|
28
|
-
|
33
|
+
twitter_stream_url = "https://stream.twitter.com/1.1/statuses/sample.json"
|
29
34
|
|
30
|
-
|
35
|
+
# Generate the HMAC-SHA1 Twitter OAuth authorization header
|
36
|
+
authorization = HttpStreamingClient::Oauth::Twitter.generate_authorization(twitter_stream_url, "get", {}, OAUTH_CONSUMER_KEY, OAUTH_CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
|
31
37
|
|
32
|
-
|
38
|
+
# Configure the client
|
39
|
+
client = HttpStreamingClient::Client.new(compression: true, reconnect: true, reconnect_interval: 10, reconnect_attempts: 60)
|
33
40
|
|
34
|
-
|
41
|
+
# Open the connection and start processing messages
|
42
|
+
response = client.get(twitter_stream_url, {:headers => {'Authorization' => "#{authorization}" }}) { |line|
|
43
|
+
logger.info "Received a line that we could parse into JSON if we want: #{line}"
|
44
|
+
client.interrupt if we_want_to_stop_receiving_messages?
|
45
|
+
}
|
46
|
+
```
|
35
47
|
|
36
|
-
|
48
|
+
For more examples, take a look at
|
37
49
|
|
38
|
-
|
50
|
+
* spec/client_spec.rb
|
51
|
+
* tools/adobe_firehose.rb
|
52
|
+
* tools/adobe_firehose_performance_test.rb
|
53
|
+
* tools/twitter_firehose.rb
|
54
|
+
* tools/twitter_firehose_performance_test.rb
|
39
55
|
|
40
|
-
|
56
|
+
## Client Configuration Options
|
41
57
|
|
42
|
-
|
58
|
+
The following options are supported as hash option parameters for HttpStreamingClient::Client.new:
|
43
59
|
|
44
|
-
|
60
|
+
GZip Compression
|
61
|
+
|
62
|
+
compression: true/false (default: true)
|
63
|
+
|
64
|
+
Automatic Socket Reconnect Functions
|
65
|
+
|
66
|
+
reconnect: true/false (default: false)
|
67
|
+
|
68
|
+
Reconnect Interval
|
69
|
+
|
70
|
+
reconnect_interval: interval_seconds (default: 1 second)
|
71
|
+
|
72
|
+
Maximum Reconnect Attempts
|
73
|
+
|
74
|
+
reconnect_attempts: num_attempts (default: 10)
|
45
75
|
|
46
76
|
## Logging
|
47
77
|
|
@@ -110,21 +140,6 @@ Individual test suites in the spec directory can be run via:
|
|
110
140
|
|
111
141
|
An HTML coverage report is generated at the end of a full test run in the coverage directory.
|
112
142
|
|
113
|
-
## Examples
|
114
|
-
|
115
|
-
Take a look at
|
116
|
-
|
117
|
-
* spec/client_spec.rb
|
118
|
-
* tools/adobe_firehose.rb
|
119
|
-
* tools/adobe_firehose_performance_test.rb
|
120
|
-
* tools/twitter_firehose.rb
|
121
|
-
* tools/twitter_firehose_performance_test.rb
|
122
|
-
|
123
|
-
## TODO
|
124
|
-
|
125
|
-
* connection management with reconnect functions
|
126
|
-
* load/memory testing
|
127
|
-
|
128
143
|
## Fixed Issues
|
129
144
|
|
130
145
|
* See [![CHANGELOG](CHANGELOG)](CHANGELOG)
|
@@ -40,10 +40,13 @@ module HttpStreamingClient
|
|
40
40
|
|
41
41
|
class Client
|
42
42
|
|
43
|
-
attr_accessor :socket, :interrupted, :compression_requested
|
43
|
+
attr_accessor :socket, :interrupted, :compression_requested, :reconnect_requested, :reconnect_attempts, :reconnect_interval
|
44
44
|
|
45
45
|
ALLOWED_MIME_TYPES = ["application/json", "text/plain", "text/html"]
|
46
46
|
|
47
|
+
MAX_RECONNECT_ATTEMPTS = 10
|
48
|
+
RECONNECT_INTERVAL_SEC = 1
|
49
|
+
|
47
50
|
def self.logger
|
48
51
|
HttpStreamingClient.logger
|
49
52
|
end
|
@@ -56,8 +59,24 @@ module HttpStreamingClient
|
|
56
59
|
logger.debug("Client.new: #{opts}")
|
57
60
|
@socket = nil
|
58
61
|
@interrupted = false
|
62
|
+
|
59
63
|
@compression_requested = opts[:compression].nil? ? true : opts[:compression]
|
60
64
|
logger.debug("compression is #{@compression_requested}")
|
65
|
+
|
66
|
+
@reconnect_requested = opts[:reconnect].nil? ? false : opts[:reconnect]
|
67
|
+
logger.debug("reconnect is #{@reconnect_requested}")
|
68
|
+
|
69
|
+
if @reconnect_requested then
|
70
|
+
@reconnect_count = 0
|
71
|
+
|
72
|
+
@reconnect_attempts = opts[:reconnect_attempts].nil? ? MAX_RECONNECT_ATTEMPTS : opts[:reconnect_attempts]
|
73
|
+
logger.debug("reconnect_attempts is #{@reconnect_attempts}")
|
74
|
+
|
75
|
+
@reconnect_interval = opts[:reconnect_interval].nil? ? RECONNECT_INTERVAL_SEC : opts[:reconnect_interval]
|
76
|
+
logger.debug("reconnect_interval is #{@reconnect_interval}")
|
77
|
+
|
78
|
+
end
|
79
|
+
|
61
80
|
end
|
62
81
|
|
63
82
|
def self.get(uri, opts = {}, &block)
|
@@ -132,199 +151,226 @@ module HttpStreamingClient
|
|
132
151
|
headers = default_headers.merge(opts[:headers] || {})
|
133
152
|
logger.debug "request headers: #{headers}"
|
134
153
|
|
135
|
-
|
136
|
-
request = "#{method} #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
|
137
|
-
request << "Host: #{uri.host}\r\n"
|
138
|
-
headers.each do |k, v|
|
139
|
-
request << "#{k}: #{v}\r\n"
|
140
|
-
end
|
141
|
-
request << "\r\n"
|
142
|
-
if method == "POST"
|
143
|
-
request << body
|
144
|
-
end
|
154
|
+
begin
|
145
155
|
|
146
|
-
|
156
|
+
socket = initialize_socket(uri, opts)
|
147
157
|
|
148
|
-
|
149
|
-
response_head[:headers] = {}
|
158
|
+
@reconnect_count = 0 if @reconnect_requested
|
150
159
|
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
header = line.split(": ")
|
156
|
-
if header.size == 1
|
157
|
-
header = header[0].split(" ")
|
158
|
-
response_head[:version] = header[0]
|
159
|
-
response_head[:code] = header[1].to_i
|
160
|
-
response_head[:msg] = header[2]
|
161
|
-
logger.debug "HTTP response code is #{response_head[:code]}"
|
162
|
-
else
|
163
|
-
response_head[:headers][camelize_header_name(header[0])] = header[1].strip
|
164
|
-
end
|
160
|
+
request = "#{method} #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
|
161
|
+
request << "Host: #{uri.host}\r\n"
|
162
|
+
headers.each do |k, v|
|
163
|
+
request << "#{k}: #{v}\r\n"
|
165
164
|
end
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
if response_head[:code] == 301 then
|
171
|
-
location = response_head[:headers]["Location"]
|
172
|
-
raise InvalidRedirect, "Unable to find Location header for HTTP 301 response" if location.nil?
|
173
|
-
logger.debug "Received HTTP 301 redirect to #{location}, following..."
|
174
|
-
socket.close if !socket.nil? and !socket.closed?
|
175
|
-
opts.delete(:socket)
|
176
|
-
return request(method, location, opts, &block)
|
177
|
-
end
|
178
|
-
|
179
|
-
content_length = response_head[:headers]["Content-Length"].to_i
|
180
|
-
logger.debug "content-length: #{content_length}"
|
181
|
-
|
182
|
-
content_type = response_head[:headers]["Content-Type"].split(';').first
|
183
|
-
logger.debug "content-type: #{content_type}"
|
184
|
-
|
185
|
-
response_compression = false
|
186
|
-
|
187
|
-
if ALLOWED_MIME_TYPES.include?(content_type)
|
188
|
-
case response_head[:headers]["Content-Encoding"]
|
189
|
-
when "gzip"
|
190
|
-
response_compression = true
|
165
|
+
request << "\r\n"
|
166
|
+
if method == "POST"
|
167
|
+
request << body
|
191
168
|
end
|
192
|
-
else
|
193
|
-
raise InvalidContentType, "invalid response MIME type: #{content_type}"
|
194
|
-
end
|
195
169
|
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
logger.debug "response: #{response}"
|
201
|
-
raise HttpError.new(response_head[:code], "Received HTTP #{response_head[:code]} response", response_head[:headers])
|
202
|
-
end
|
170
|
+
socket.write(request)
|
171
|
+
|
172
|
+
response_head = {}
|
173
|
+
response_head[:headers] = {}
|
203
174
|
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
response = ""
|
208
|
-
|
209
|
-
if response_compression then
|
210
|
-
logger.debug "response compression detected"
|
211
|
-
if block_given? then
|
212
|
-
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
213
|
-
logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
|
214
|
-
block.call(line) unless @interrupted }
|
175
|
+
socket.each_line do |line|
|
176
|
+
if line == "\r\n" then
|
177
|
+
break
|
215
178
|
else
|
216
|
-
|
217
|
-
|
218
|
-
|
179
|
+
header = line.split(": ")
|
180
|
+
if header.size == 1
|
181
|
+
header = header[0].split(" ")
|
182
|
+
response_head[:version] = header[0]
|
183
|
+
response_head[:code] = header[1].to_i
|
184
|
+
response_head[:msg] = header[2]
|
185
|
+
logger.debug "HTTP response code is #{response_head[:code]}"
|
186
|
+
else
|
187
|
+
response_head[:headers][camelize_header_name(header[0])] = header[1].strip
|
188
|
+
end
|
219
189
|
end
|
220
190
|
end
|
221
191
|
|
222
|
-
|
223
|
-
chunkLeft = 0
|
192
|
+
logger.debug "response headers:#{response_head[:headers]}"
|
224
193
|
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
194
|
+
if response_head[:code] == 301 then
|
195
|
+
location = response_head[:headers]["Location"]
|
196
|
+
raise InvalidRedirect, "Unable to find Location header for HTTP 301 response" if location.nil?
|
197
|
+
logger.debug "Received HTTP 301 redirect to #{location}, following..."
|
198
|
+
socket.close if !socket.nil? and !socket.closed?
|
199
|
+
opts.delete(:socket)
|
200
|
+
return request(method, location, opts, &block)
|
201
|
+
end
|
230
202
|
|
231
|
-
|
203
|
+
content_length = response_head[:headers]["Content-Length"].to_i
|
204
|
+
logger.debug "content-length: #{content_length}"
|
232
205
|
|
233
|
-
|
234
|
-
|
206
|
+
content_type = response_head[:headers]["Content-Type"].split(';').first
|
207
|
+
logger.debug "content-type: #{content_type}"
|
235
208
|
|
236
|
-
|
237
|
-
next if partial.nil?
|
209
|
+
response_compression = false
|
238
210
|
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
remaining = size-partial.size
|
244
|
-
logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
|
211
|
+
if ALLOWED_MIME_TYPES.include?(content_type)
|
212
|
+
case response_head[:headers]["Content-Encoding"]
|
213
|
+
when "gzip"
|
214
|
+
response_compression = true
|
245
215
|
end
|
216
|
+
else
|
217
|
+
raise InvalidContentType, "invalid response MIME type: #{content_type}"
|
218
|
+
end
|
219
|
+
|
220
|
+
if (response_head[:code] != 200)
|
221
|
+
s = "Received HTTP #{response_head[:code]} response"
|
222
|
+
logger.debug "request: #{request}"
|
223
|
+
response = socket.read(content_length)
|
224
|
+
logger.debug "response: #{response}"
|
225
|
+
raise HttpError.new(response_head[:code], "Received HTTP #{response_head[:code]} response", response_head[:headers])
|
226
|
+
end
|
246
227
|
|
247
|
-
|
228
|
+
if response_head[:headers]["Transfer-Encoding"] == 'chunked'
|
229
|
+
partial = nil
|
230
|
+
decoder = nil
|
231
|
+
response = ""
|
248
232
|
|
249
233
|
if response_compression then
|
250
|
-
|
251
|
-
else
|
234
|
+
logger.debug "response compression detected"
|
252
235
|
if block_given? then
|
253
|
-
|
236
|
+
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
237
|
+
logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
|
238
|
+
block.call(line) unless @interrupted }
|
254
239
|
else
|
255
|
-
|
256
|
-
|
240
|
+
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
241
|
+
logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
|
242
|
+
response << line unless @interrupted }
|
257
243
|
end
|
258
244
|
end
|
259
|
-
end
|
260
245
|
|
261
|
-
|
262
|
-
|
263
|
-
else
|
264
|
-
# Not chunked transfer encoding, but potentially gzip'd, and potentially streaming with content-length = 0
|
265
|
-
|
266
|
-
if content_length > 0 then
|
267
|
-
bits = socket.read(content_length)
|
268
|
-
logger.debug "read #{content_length} bytes"
|
269
|
-
return bits if !response_compression
|
270
|
-
logger.debug "response compression detected"
|
271
|
-
begin
|
272
|
-
decoder = Zlib::GzipReader.new(StringIO.new(bits))
|
273
|
-
return decoder.read
|
274
|
-
rescue Zlib::Error
|
275
|
-
raise DecoderError
|
276
|
-
end
|
277
|
-
end
|
246
|
+
while !socket.eof? && (line = socket.gets)
|
247
|
+
chunkLeft = 0
|
278
248
|
|
279
|
-
|
249
|
+
if line.match /^0\r\n/ then
|
250
|
+
logger.debug "received zero length chunk, chunked encoding EOF"
|
251
|
+
logger.debug "EOF line: #{line}"
|
252
|
+
break
|
253
|
+
end
|
280
254
|
|
281
|
-
|
282
|
-
decoder = nil
|
283
|
-
response = ""
|
255
|
+
next if line == "\r\n"
|
284
256
|
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
257
|
+
size = line.hex
|
258
|
+
logger.debug "chunk size:#{size}"
|
259
|
+
|
260
|
+
partial = socket.read(size)
|
261
|
+
next if partial.nil?
|
262
|
+
|
263
|
+
remaining = size-partial.size
|
264
|
+
logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
|
265
|
+
until remaining == 0
|
266
|
+
partial << socket.read(remaining)
|
267
|
+
remaining = size-partial.size
|
268
|
+
logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
|
269
|
+
end
|
270
|
+
|
271
|
+
return if @interrupted
|
294
272
|
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
273
|
+
if response_compression then
|
274
|
+
decoder << partial
|
275
|
+
else
|
276
|
+
if block_given? then
|
277
|
+
yield partial
|
278
|
+
else
|
279
|
+
logger.debug "no block specified, returning chunk results and halting streaming response"
|
280
|
+
response << partial
|
281
|
+
interrupt
|
282
|
+
end
|
283
|
+
end
|
299
284
|
end
|
300
|
-
logger.debug "EOF detected"
|
301
|
-
decoder = nil
|
302
285
|
|
286
|
+
logger.debug "socket EOF detected" if socket.eof?
|
287
|
+
raise ReconnectRequest if @reconnect_requested
|
303
288
|
return response
|
304
289
|
|
305
290
|
else
|
291
|
+
# Not chunked transfer encoding, but potentially gzip'd, and potentially streaming with content-length = 0
|
292
|
+
|
293
|
+
if content_length > 0 then
|
294
|
+
bits = socket.read(content_length)
|
295
|
+
logger.debug "read #{content_length} bytes"
|
296
|
+
return bits if !response_compression
|
297
|
+
logger.debug "response compression detected"
|
298
|
+
begin
|
299
|
+
decoder = Zlib::GzipReader.new(StringIO.new(bits))
|
300
|
+
return decoder.read
|
301
|
+
rescue Zlib::Error
|
302
|
+
raise DecoderError
|
303
|
+
end
|
304
|
+
end
|
306
305
|
|
307
|
-
|
306
|
+
if response_compression then
|
307
|
+
|
308
|
+
logger.debug "response compression detected"
|
309
|
+
decoder = nil
|
310
|
+
response = ""
|
308
311
|
|
309
|
-
while (!socket.eof? and !(line = socket.readline).nil?)
|
310
312
|
if block_given? then
|
311
|
-
|
312
|
-
|
313
|
+
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
314
|
+
logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
|
315
|
+
block.call(line) unless @interrupted }
|
313
316
|
else
|
314
|
-
|
315
|
-
|
317
|
+
decoder = HttpStreamingClient::Decoders::GZip.new { |line|
|
318
|
+
logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
|
319
|
+
response << line unless @interrupted }
|
316
320
|
end
|
317
|
-
break if @interrupted
|
318
|
-
end
|
319
321
|
|
320
|
-
|
322
|
+
while (!socket.eof? and !(line = socket.read_nonblock(2048)).nil?)
|
323
|
+
logger.debug "read compressed line, #{line.size} bytes"
|
324
|
+
decoder << line
|
325
|
+
break response if @interrupted
|
326
|
+
end
|
327
|
+
|
328
|
+
logger.debug "EOF detected"
|
329
|
+
raise ReconnectRequest if @reconnect_requested
|
330
|
+
return response
|
331
|
+
|
332
|
+
else
|
333
|
+
|
334
|
+
response = ""
|
335
|
+
|
336
|
+
while (!socket.eof? and !(line = socket.readline).nil?)
|
337
|
+
if block_given? then
|
338
|
+
yield line
|
339
|
+
logger.debug "read #{line.size} bytes"
|
340
|
+
else
|
341
|
+
logger.debug "read #{line.size} bytes, #{response.size} bytes total"
|
342
|
+
response << line
|
343
|
+
end
|
344
|
+
break if @interrupted
|
345
|
+
end
|
346
|
+
|
347
|
+
raise ReconnectRequest if @reconnect_requested
|
348
|
+
return response
|
349
|
+
|
350
|
+
end
|
351
|
+
end
|
352
|
+
rescue => e
|
353
|
+
return if @interrupted
|
354
|
+
logger.debug "Error Detected: #{e}" unless e.instance_of? ReconnectRequest
|
355
|
+
decoder.close if !decoder.nil?
|
356
|
+
socket.close if !socket.nil? and !socket.closed?
|
357
|
+
opts.delete(:socket)
|
321
358
|
|
359
|
+
if @reconnect_requested then
|
360
|
+
logger.info "Connection closed. Reconnect requested. Trying..."
|
361
|
+
@reconnect_count = @reconnect_count + 1
|
362
|
+
sleep @reconnect_interval
|
363
|
+
retry if @reconnect_count < @reconnect_attempts
|
364
|
+
logger.info "Maximum number of failed reconnect attempts reached (#{@reconnect_attempts}). Exiting."
|
322
365
|
end
|
366
|
+
|
367
|
+
raise e unless e.instance_of? ReconnectRequest
|
323
368
|
end
|
324
369
|
ensure
|
325
370
|
logger.debug "ensure socket closed"
|
326
371
|
decoder.close if !decoder.nil?
|
327
372
|
socket.close if !socket.nil? and !socket.closed?
|
373
|
+
opts.delete(:socket)
|
328
374
|
end
|
329
375
|
|
330
376
|
private
|
@@ -340,6 +386,8 @@ module HttpStreamingClient
|
|
340
386
|
uri = URI.parse(uri)
|
341
387
|
end
|
342
388
|
|
389
|
+
logger.debug "Connecting to #{uri.host}:#{uri.port}..."
|
390
|
+
|
343
391
|
@socket = TCPSocket.new(uri.host, uri.port)
|
344
392
|
|
345
393
|
if uri.scheme.eql? "https"
|
@@ -351,6 +399,7 @@ module HttpStreamingClient
|
|
351
399
|
end
|
352
400
|
end
|
353
401
|
|
402
|
+
logger.debug "Connected to #{uri.host}:#{uri.port}"
|
354
403
|
opts.merge!({:socket => @socket})
|
355
404
|
@interrupted = false
|
356
405
|
return opts[:socket]
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
|
4
|
+
describe HttpStreamingClient do
|
5
|
+
|
6
|
+
# currently disabled, requires a server that can be killed to simulate dropped connections
|
7
|
+
|
8
|
+
it "should receive exactly 10 messages, no reconnect" do
|
9
|
+
#it "should receive exactly 10 messages, no reconnect", :disabled => true do
|
10
|
+
|
11
|
+
count = 0
|
12
|
+
client = HttpStreamingClient::Client.new(compression: false)
|
13
|
+
response = client.get("http://localhost:3000/outbounds/consumer/1") { |line|
|
14
|
+
logger.debug "line received: #{line}"
|
15
|
+
count = count + 1
|
16
|
+
}
|
17
|
+
expect(response).to be_true
|
18
|
+
expect(count).to be(10)
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should reconnect on any error or EOF" do
|
22
|
+
|
23
|
+
client = HttpStreamingClient::Client.new(compression: false, reconnect: true, reconnect_attempts: 5, reconnect_interval: 1)
|
24
|
+
count = 0
|
25
|
+
response = client.get("http://localhost:3000/outbounds/consumer/1") { |line|
|
26
|
+
logger.debug "line received: #{line}"
|
27
|
+
count = count + 1
|
28
|
+
client.interrupt if count > 20
|
29
|
+
}
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -15,8 +15,8 @@ authorization = HttpStreamingClient::Oauth::Adobe.generate_authorization(url, US
|
|
15
15
|
puts "#{TOKENAPIHOST}:access token: #{authorization}"
|
16
16
|
client = HttpStreamingClient::Client.new(compression: false)
|
17
17
|
|
18
|
-
NUM_RECORDS_PER_BATCH =
|
19
|
-
MAX_RECORDS =
|
18
|
+
NUM_RECORDS_PER_BATCH = 5000
|
19
|
+
MAX_RECORDS = 3600000
|
20
20
|
|
21
21
|
count = 0
|
22
22
|
totalSize = 0
|
@@ -25,6 +25,7 @@ startTime = nil
|
|
25
25
|
lastTime = nil
|
26
26
|
|
27
27
|
puts "starting performance test run: #{Time.new.to_s}"
|
28
|
+
puts "stream: #{STREAMURL}"
|
28
29
|
|
29
30
|
startTime = lastTime = Time.new
|
30
31
|
|
@@ -69,6 +70,7 @@ response = client.get(STREAMURL, {:headers => {'Authorization' => "Bearer #{auth
|
|
69
70
|
stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
|
70
71
|
|
71
72
|
puts stats.to_json
|
73
|
+
STDOUT.flush
|
72
74
|
|
73
75
|
lastTime = now
|
74
76
|
intervalSize = 0
|
@@ -24,6 +24,7 @@ startTime = nil
|
|
24
24
|
lastTime = nil
|
25
25
|
|
26
26
|
puts "starting performance test run: #{Time.new.to_s}"
|
27
|
+
puts "stream: #{url}"
|
27
28
|
|
28
29
|
startTime = lastTime = Time.new
|
29
30
|
|
@@ -64,6 +65,7 @@ response = client.get(url, {:headers => {'Authorization' => "#{authorization}" }
|
|
64
65
|
stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
|
65
66
|
|
66
67
|
puts stats.to_json
|
68
|
+
STDOUT.flush
|
67
69
|
|
68
70
|
lastTime = now
|
69
71
|
intervalSize = 0
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: http_streaming_client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Tompkins
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-01-
|
11
|
+
date: 2014-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -128,6 +128,7 @@ files:
|
|
128
128
|
- spec/adobe_spec.rb
|
129
129
|
- spec/client_spec.rb
|
130
130
|
- spec/oauth_spec.rb
|
131
|
+
- spec/reconnect_spec.rb
|
131
132
|
- spec/spec_helper.rb
|
132
133
|
- spec/twitter_spec.rb
|
133
134
|
- tools/adobe_firehose.rb
|
@@ -164,5 +165,6 @@ test_files:
|
|
164
165
|
- spec/adobe_spec.rb
|
165
166
|
- spec/client_spec.rb
|
166
167
|
- spec/oauth_spec.rb
|
168
|
+
- spec/reconnect_spec.rb
|
167
169
|
- spec/spec_helper.rb
|
168
170
|
- spec/twitter_spec.rb
|