http_streaming_client 0.8.6 → 0.8.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 607b852759a7216b85fa6d820d2bc70a2d50b475
4
- data.tar.gz: 49b554701c78b290e3b7e337a1b8fbcc12f0f726
3
+ metadata.gz: 8308d062ebb595dddc19e1ce5232829fb92bd9ce
4
+ data.tar.gz: cfbe2c60d0b7b021cbe09d80bee4ef8114115039
5
5
  SHA512:
6
- metadata.gz: c1d07b0e483dc0e81d83800d05d6170d28f00dad305f6e6f3644ffe96b9bac00ff203a7bd34fadf697245b10520cdb1e303b79442dc0bd6bd3a477bb52b99f63
7
- data.tar.gz: a26cf37f5791d4701ca0f0bf74cdedb009fb1a6d1ee58934f15dfad8a12ffb8c3755a3854c95b496fc2ca22bae69c70e8a04d40c7ada2efd54f41c5ead6ca2d2
6
+ metadata.gz: 50ee772c4e19aec32f3a771b1686b8beadd9d7fb2038cf08b0b65792f7b070f26b1bf74519c4f85deea9462e2a93df0939888d3c1233a383a6b21286ce93a6c1
7
+ data.tar.gz: 323dec68b2c4d8730d4c9803358f142e721efc3aa2b08745625eacd4ea6b59351e2394a24ca739d5baa5ef64f1343f2060d538a7d3bc2c7f3b3fdded0f17d066
data/CHANGELOG CHANGED
@@ -1,3 +1,11 @@
1
+ http_streaming_client 0.8.8 (1.7.2014)
2
+ ========================================
3
+ * Fixed Issue #1, memory leak due to accumulated messages within the buffer pipeline
4
+ * Added automatic reconnection support and configuration options
5
+ * Added automatic HTTP 301 redirect support
6
+ * Added new unit tests for reconnect and for memory leak detection
7
+ * Updated performance tests
8
+
1
9
  http_streaming_client 0.8.5 (12.10.2013)
2
10
  ========================================
3
11
  * Updates to support MRI ruby 2.0.0-p353
data/README.md CHANGED
@@ -9,39 +9,69 @@ Ruby HTTP client with support for HTTP 1.1 streaming, GZIP and zlib compressed s
9
9
 
10
10
  MRI ruby-2.0.0-p353. If you need it, install via rvm: https://rvm.io/
11
11
 
12
- ## Installation (local gem bundle/install)
12
+ ## Installation
13
13
 
14
- Execute the following to bundle the gem:
14
+ Add this line to your application's Gemfile:
15
15
 
16
- $ gem build http_streaming_client.gemspec
16
+ gem 'http_streaming_client'
17
17
 
18
- Then install the gem with:
18
+ And then execute:
19
+
20
+ $ bundle install
21
+
22
+ Or install it directly via gem with:
19
23
 
20
24
  $ gem install http_streaming_client
21
25
 
22
- ## Installation (via Github)
26
+ ## Simple Example
23
27
 
24
- Add this line to your application's Gemfile:
28
+ Twitter Firehose Sample Stream:
25
29
 
26
- gem 'http_streaming_client', :git => 'git@github.com:adobe-research/http_streaming_client.git'
30
+ ```ruby
31
+ require 'http_streaming_client'
27
32
 
28
- And then execute:
33
+ twitter_stream_url = "https://stream.twitter.com/1.1/statuses/sample.json"
29
34
 
30
- $ bundle install
35
+ # Generate the HMAC-SHA1 Twitter OAuth authorization header
36
+ authorization = HttpStreamingClient::Oauth::Twitter.generate_authorization(twitter_stream_url, "get", {}, OAUTH_CONSUMER_KEY, OAUTH_CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
31
37
 
32
- ## Installation (via rubygems.org)
38
+ # Configure the client
39
+ client = HttpStreamingClient::Client.new(compression: true, reconnect: true, reconnect_interval: 10, reconnect_attempts: 60)
33
40
 
34
- Add this line to your application's Gemfile:
41
+ # Open the connection and start processing messages
42
+ response = client.get(twitter_stream_url, {:headers => {'Authorization' => "#{authorization}" }}) { |line|
43
+ logger.info "Received a line that we could parse into JSON if we want: #{line}"
44
+ client.interrupt if we_want_to_stop_receiving_messages?
45
+ }
46
+ ```
35
47
 
36
- gem 'http_streaming_client'
48
+ For more examples, take a look at
37
49
 
38
- And then execute:
50
+ * spec/client_spec.rb
51
+ * tools/adobe_firehose.rb
52
+ * tools/adobe_firehose_performance_test.rb
53
+ * tools/twitter_firehose.rb
54
+ * tools/twitter_firehose_performance_test.rb
39
55
 
40
- $ bundle
56
+ ## Client Configuration Options
41
57
 
42
- Or install it yourself as:
58
+ The following options are supported as hash option parameters for HttpStreamingClient::Client.new:
43
59
 
44
- $ gem install http_streaming_client
60
+ GZip Compression
61
+
62
+ compression: true/false (default: true)
63
+
64
+ Automatic Socket Reconnect Functions
65
+
66
+ reconnect: true/false (default: false)
67
+
68
+ Reconnect Interval
69
+
70
+ reconnect_interval: interval_seconds (default: 1 second)
71
+
72
+ Maximum Reconnect Attempts
73
+
74
+ reconnect_attempts: num_attempts (default: 10)
45
75
 
46
76
  ## Logging
47
77
 
@@ -110,21 +140,6 @@ Individual test suites in the spec directory can be run via:
110
140
 
111
141
  An HTML coverage report is generated at the end of a full test run in the coverage directory.
112
142
 
113
- ## Examples
114
-
115
- Take a look at
116
-
117
- * spec/client_spec.rb
118
- * tools/adobe_firehose.rb
119
- * tools/adobe_firehose_performance_test.rb
120
- * tools/twitter_firehose.rb
121
- * tools/twitter_firehose_performance_test.rb
122
-
123
- ## TODO
124
-
125
- * connection management with reconnect functions
126
- * load/memory testing
127
-
128
143
  ## Fixed Issues
129
144
 
130
145
  * See [![CHANGELOG](CHANGELOG)](CHANGELOG)
@@ -40,10 +40,13 @@ module HttpStreamingClient
40
40
 
41
41
  class Client
42
42
 
43
- attr_accessor :socket, :interrupted, :compression_requested
43
+ attr_accessor :socket, :interrupted, :compression_requested, :reconnect_requested, :reconnect_attempts, :reconnect_interval
44
44
 
45
45
  ALLOWED_MIME_TYPES = ["application/json", "text/plain", "text/html"]
46
46
 
47
+ MAX_RECONNECT_ATTEMPTS = 10
48
+ RECONNECT_INTERVAL_SEC = 1
49
+
47
50
  def self.logger
48
51
  HttpStreamingClient.logger
49
52
  end
@@ -56,8 +59,24 @@ module HttpStreamingClient
56
59
  logger.debug("Client.new: #{opts}")
57
60
  @socket = nil
58
61
  @interrupted = false
62
+
59
63
  @compression_requested = opts[:compression].nil? ? true : opts[:compression]
60
64
  logger.debug("compression is #{@compression_requested}")
65
+
66
+ @reconnect_requested = opts[:reconnect].nil? ? false : opts[:reconnect]
67
+ logger.debug("reconnect is #{@reconnect_requested}")
68
+
69
+ if @reconnect_requested then
70
+ @reconnect_count = 0
71
+
72
+ @reconnect_attempts = opts[:reconnect_attempts].nil? ? MAX_RECONNECT_ATTEMPTS : opts[:reconnect_attempts]
73
+ logger.debug("reconnect_attempts is #{@reconnect_attempts}")
74
+
75
+ @reconnect_interval = opts[:reconnect_interval].nil? ? RECONNECT_INTERVAL_SEC : opts[:reconnect_interval]
76
+ logger.debug("reconnect_interval is #{@reconnect_interval}")
77
+
78
+ end
79
+
61
80
  end
62
81
 
63
82
  def self.get(uri, opts = {}, &block)
@@ -132,199 +151,226 @@ module HttpStreamingClient
132
151
  headers = default_headers.merge(opts[:headers] || {})
133
152
  logger.debug "request headers: #{headers}"
134
153
 
135
- socket = initialize_socket(uri, opts)
136
- request = "#{method} #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
137
- request << "Host: #{uri.host}\r\n"
138
- headers.each do |k, v|
139
- request << "#{k}: #{v}\r\n"
140
- end
141
- request << "\r\n"
142
- if method == "POST"
143
- request << body
144
- end
154
+ begin
145
155
 
146
- socket.write(request)
156
+ socket = initialize_socket(uri, opts)
147
157
 
148
- response_head = {}
149
- response_head[:headers] = {}
158
+ @reconnect_count = 0 if @reconnect_requested
150
159
 
151
- socket.each_line do |line|
152
- if line == "\r\n" then
153
- break
154
- else
155
- header = line.split(": ")
156
- if header.size == 1
157
- header = header[0].split(" ")
158
- response_head[:version] = header[0]
159
- response_head[:code] = header[1].to_i
160
- response_head[:msg] = header[2]
161
- logger.debug "HTTP response code is #{response_head[:code]}"
162
- else
163
- response_head[:headers][camelize_header_name(header[0])] = header[1].strip
164
- end
160
+ request = "#{method} #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
161
+ request << "Host: #{uri.host}\r\n"
162
+ headers.each do |k, v|
163
+ request << "#{k}: #{v}\r\n"
165
164
  end
166
- end
167
-
168
- logger.debug "response headers:#{response_head[:headers]}"
169
-
170
- if response_head[:code] == 301 then
171
- location = response_head[:headers]["Location"]
172
- raise InvalidRedirect, "Unable to find Location header for HTTP 301 response" if location.nil?
173
- logger.debug "Received HTTP 301 redirect to #{location}, following..."
174
- socket.close if !socket.nil? and !socket.closed?
175
- opts.delete(:socket)
176
- return request(method, location, opts, &block)
177
- end
178
-
179
- content_length = response_head[:headers]["Content-Length"].to_i
180
- logger.debug "content-length: #{content_length}"
181
-
182
- content_type = response_head[:headers]["Content-Type"].split(';').first
183
- logger.debug "content-type: #{content_type}"
184
-
185
- response_compression = false
186
-
187
- if ALLOWED_MIME_TYPES.include?(content_type)
188
- case response_head[:headers]["Content-Encoding"]
189
- when "gzip"
190
- response_compression = true
165
+ request << "\r\n"
166
+ if method == "POST"
167
+ request << body
191
168
  end
192
- else
193
- raise InvalidContentType, "invalid response MIME type: #{content_type}"
194
- end
195
169
 
196
- if (response_head[:code] != 200)
197
- s = "Received HTTP #{response_head[:code]} response"
198
- logger.debug "request: #{request}"
199
- response = socket.read(content_length)
200
- logger.debug "response: #{response}"
201
- raise HttpError.new(response_head[:code], "Received HTTP #{response_head[:code]} response", response_head[:headers])
202
- end
170
+ socket.write(request)
171
+
172
+ response_head = {}
173
+ response_head[:headers] = {}
203
174
 
204
- if response_head[:headers]["Transfer-Encoding"] == 'chunked'
205
- partial = nil
206
- decoder = nil
207
- response = ""
208
-
209
- if response_compression then
210
- logger.debug "response compression detected"
211
- if block_given? then
212
- decoder = HttpStreamingClient::Decoders::GZip.new { |line|
213
- logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
214
- block.call(line) unless @interrupted }
175
+ socket.each_line do |line|
176
+ if line == "\r\n" then
177
+ break
215
178
  else
216
- decoder = HttpStreamingClient::Decoders::GZip.new { |line|
217
- logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
218
- response << line unless @interrupted }
179
+ header = line.split(": ")
180
+ if header.size == 1
181
+ header = header[0].split(" ")
182
+ response_head[:version] = header[0]
183
+ response_head[:code] = header[1].to_i
184
+ response_head[:msg] = header[2]
185
+ logger.debug "HTTP response code is #{response_head[:code]}"
186
+ else
187
+ response_head[:headers][camelize_header_name(header[0])] = header[1].strip
188
+ end
219
189
  end
220
190
  end
221
191
 
222
- while !socket.eof? && (line = socket.gets)
223
- chunkLeft = 0
192
+ logger.debug "response headers:#{response_head[:headers]}"
224
193
 
225
- if line.match /^0\r\n/ then
226
- logger.debug "received zero length chunk, chunked encoding EOF"
227
- logger.debug "EOF line: #{line}"
228
- break
229
- end
194
+ if response_head[:code] == 301 then
195
+ location = response_head[:headers]["Location"]
196
+ raise InvalidRedirect, "Unable to find Location header for HTTP 301 response" if location.nil?
197
+ logger.debug "Received HTTP 301 redirect to #{location}, following..."
198
+ socket.close if !socket.nil? and !socket.closed?
199
+ opts.delete(:socket)
200
+ return request(method, location, opts, &block)
201
+ end
230
202
 
231
- next if line == "\r\n"
203
+ content_length = response_head[:headers]["Content-Length"].to_i
204
+ logger.debug "content-length: #{content_length}"
232
205
 
233
- size = line.hex
234
- logger.debug "chunk size:#{size}"
206
+ content_type = response_head[:headers]["Content-Type"].split(';').first
207
+ logger.debug "content-type: #{content_type}"
235
208
 
236
- partial = socket.read(size)
237
- next if partial.nil?
209
+ response_compression = false
238
210
 
239
- remaining = size-partial.size
240
- logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
241
- until remaining == 0
242
- partial << socket.read(remaining)
243
- remaining = size-partial.size
244
- logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
211
+ if ALLOWED_MIME_TYPES.include?(content_type)
212
+ case response_head[:headers]["Content-Encoding"]
213
+ when "gzip"
214
+ response_compression = true
245
215
  end
216
+ else
217
+ raise InvalidContentType, "invalid response MIME type: #{content_type}"
218
+ end
219
+
220
+ if (response_head[:code] != 200)
221
+ s = "Received HTTP #{response_head[:code]} response"
222
+ logger.debug "request: #{request}"
223
+ response = socket.read(content_length)
224
+ logger.debug "response: #{response}"
225
+ raise HttpError.new(response_head[:code], "Received HTTP #{response_head[:code]} response", response_head[:headers])
226
+ end
246
227
 
247
- return if @interrupted
228
+ if response_head[:headers]["Transfer-Encoding"] == 'chunked'
229
+ partial = nil
230
+ decoder = nil
231
+ response = ""
248
232
 
249
233
  if response_compression then
250
- decoder << partial
251
- else
234
+ logger.debug "response compression detected"
252
235
  if block_given? then
253
- yield partial
236
+ decoder = HttpStreamingClient::Decoders::GZip.new { |line|
237
+ logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
238
+ block.call(line) unless @interrupted }
254
239
  else
255
- logger.debug "no block specified, returning chunk results and halting streaming response"
256
- response << partial
240
+ decoder = HttpStreamingClient::Decoders::GZip.new { |line|
241
+ logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
242
+ response << line unless @interrupted }
257
243
  end
258
244
  end
259
- end
260
245
 
261
- return response
262
-
263
- else
264
- # Not chunked transfer encoding, but potentially gzip'd, and potentially streaming with content-length = 0
265
-
266
- if content_length > 0 then
267
- bits = socket.read(content_length)
268
- logger.debug "read #{content_length} bytes"
269
- return bits if !response_compression
270
- logger.debug "response compression detected"
271
- begin
272
- decoder = Zlib::GzipReader.new(StringIO.new(bits))
273
- return decoder.read
274
- rescue Zlib::Error
275
- raise DecoderError
276
- end
277
- end
246
+ while !socket.eof? && (line = socket.gets)
247
+ chunkLeft = 0
278
248
 
279
- if response_compression then
249
+ if line.match /^0\r\n/ then
250
+ logger.debug "received zero length chunk, chunked encoding EOF"
251
+ logger.debug "EOF line: #{line}"
252
+ break
253
+ end
280
254
 
281
- logger.debug "response compression detected"
282
- decoder = nil
283
- response = ""
255
+ next if line == "\r\n"
284
256
 
285
- if block_given? then
286
- decoder = HttpStreamingClient::Decoders::GZip.new { |line|
287
- logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
288
- block.call(line) unless @interrupted }
289
- else
290
- decoder = HttpStreamingClient::Decoders::GZip.new { |line|
291
- logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
292
- response << line unless @interrupted }
293
- end
257
+ size = line.hex
258
+ logger.debug "chunk size:#{size}"
259
+
260
+ partial = socket.read(size)
261
+ next if partial.nil?
262
+
263
+ remaining = size-partial.size
264
+ logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
265
+ until remaining == 0
266
+ partial << socket.read(remaining)
267
+ remaining = size-partial.size
268
+ logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
269
+ end
270
+
271
+ return if @interrupted
294
272
 
295
- while (!socket.eof? and !(line = socket.read_nonblock(2048)).nil?)
296
- logger.debug "read compressed line, #{line.size} bytes"
297
- decoder << line
298
- break response if @interrupted
273
+ if response_compression then
274
+ decoder << partial
275
+ else
276
+ if block_given? then
277
+ yield partial
278
+ else
279
+ logger.debug "no block specified, returning chunk results and halting streaming response"
280
+ response << partial
281
+ interrupt
282
+ end
283
+ end
299
284
  end
300
- logger.debug "EOF detected"
301
- decoder = nil
302
285
 
286
+ logger.debug "socket EOF detected" if socket.eof?
287
+ raise ReconnectRequest if @reconnect_requested
303
288
  return response
304
289
 
305
290
  else
291
+ # Not chunked transfer encoding, but potentially gzip'd, and potentially streaming with content-length = 0
292
+
293
+ if content_length > 0 then
294
+ bits = socket.read(content_length)
295
+ logger.debug "read #{content_length} bytes"
296
+ return bits if !response_compression
297
+ logger.debug "response compression detected"
298
+ begin
299
+ decoder = Zlib::GzipReader.new(StringIO.new(bits))
300
+ return decoder.read
301
+ rescue Zlib::Error
302
+ raise DecoderError
303
+ end
304
+ end
306
305
 
307
- response = ""
306
+ if response_compression then
307
+
308
+ logger.debug "response compression detected"
309
+ decoder = nil
310
+ response = ""
308
311
 
309
- while (!socket.eof? and !(line = socket.readline).nil?)
310
312
  if block_given? then
311
- yield line
312
- logger.debug "read #{line.size} bytes"
313
+ decoder = HttpStreamingClient::Decoders::GZip.new { |line|
314
+ logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
315
+ block.call(line) unless @interrupted }
313
316
  else
314
- logger.debug "read #{line.size} bytes, #{response.size} bytes total"
315
- response << line
317
+ decoder = HttpStreamingClient::Decoders::GZip.new { |line|
318
+ logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
319
+ response << line unless @interrupted }
316
320
  end
317
- break if @interrupted
318
- end
319
321
 
320
- return response
322
+ while (!socket.eof? and !(line = socket.read_nonblock(2048)).nil?)
323
+ logger.debug "read compressed line, #{line.size} bytes"
324
+ decoder << line
325
+ break response if @interrupted
326
+ end
327
+
328
+ logger.debug "EOF detected"
329
+ raise ReconnectRequest if @reconnect_requested
330
+ return response
331
+
332
+ else
333
+
334
+ response = ""
335
+
336
+ while (!socket.eof? and !(line = socket.readline).nil?)
337
+ if block_given? then
338
+ yield line
339
+ logger.debug "read #{line.size} bytes"
340
+ else
341
+ logger.debug "read #{line.size} bytes, #{response.size} bytes total"
342
+ response << line
343
+ end
344
+ break if @interrupted
345
+ end
346
+
347
+ raise ReconnectRequest if @reconnect_requested
348
+ return response
349
+
350
+ end
351
+ end
352
+ rescue => e
353
+ return if @interrupted
354
+ logger.debug "Error Detected: #{e}" unless e.instance_of? ReconnectRequest
355
+ decoder.close if !decoder.nil?
356
+ socket.close if !socket.nil? and !socket.closed?
357
+ opts.delete(:socket)
321
358
 
359
+ if @reconnect_requested then
360
+ logger.info "Connection closed. Reconnect requested. Trying..."
361
+ @reconnect_count = @reconnect_count + 1
362
+ sleep @reconnect_interval
363
+ retry if @reconnect_count < @reconnect_attempts
364
+ logger.info "Maximum number of failed reconnect attempts reached (#{@reconnect_attempts}). Exiting."
322
365
  end
366
+
367
+ raise e unless e.instance_of? ReconnectRequest
323
368
  end
324
369
  ensure
325
370
  logger.debug "ensure socket closed"
326
371
  decoder.close if !decoder.nil?
327
372
  socket.close if !socket.nil? and !socket.closed?
373
+ opts.delete(:socket)
328
374
  end
329
375
 
330
376
  private
@@ -340,6 +386,8 @@ module HttpStreamingClient
340
386
  uri = URI.parse(uri)
341
387
  end
342
388
 
389
+ logger.debug "Connecting to #{uri.host}:#{uri.port}..."
390
+
343
391
  @socket = TCPSocket.new(uri.host, uri.port)
344
392
 
345
393
  if uri.scheme.eql? "https"
@@ -351,6 +399,7 @@ module HttpStreamingClient
351
399
  end
352
400
  end
353
401
 
402
+ logger.debug "Connected to #{uri.host}:#{uri.port}"
354
403
  opts.merge!({:socket => @socket})
355
404
  @interrupted = false
356
405
  return opts[:socket]
@@ -105,7 +105,7 @@ module HttpStreamingClient
105
105
 
106
106
  protected
107
107
 
108
- class GZipBufferIO
108
+ class GZipBufferIO < IO
109
109
 
110
110
  def logger
111
111
  HttpStreamingClient.logger
@@ -29,9 +29,12 @@
29
29
 
30
30
  module HttpStreamingClient
31
31
 
32
+
32
33
  class InvalidContentType < Exception; end
33
34
 
34
35
  class InvalidRedirect < Exception; end
36
+
37
+ class ReconnectRequest < StandardError; end
35
38
 
36
39
  class HttpTimeOut < StandardError; end
37
40
 
@@ -28,5 +28,5 @@
28
28
  ###########################################################################
29
29
 
30
30
  module HttpStreamingClient
31
- VERSION = "0.8.6"
31
+ VERSION = "0.8.8"
32
32
  end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper'
2
+
3
+
4
+ describe HttpStreamingClient do
5
+
6
+ # currently disabled, requires a server that can be killed to simulate dropped connections
7
+
8
+ it "should receive exactly 10 messages, no reconnect" do
9
+ #it "should receive exactly 10 messages, no reconnect", :disabled => true do
10
+
11
+ count = 0
12
+ client = HttpStreamingClient::Client.new(compression: false)
13
+ response = client.get("http://localhost:3000/outbounds/consumer/1") { |line|
14
+ logger.debug "line received: #{line}"
15
+ count = count + 1
16
+ }
17
+ expect(response).to be_true
18
+ expect(count).to be(10)
19
+ end
20
+
21
+ it "should reconnect on any error or EOF" do
22
+
23
+ client = HttpStreamingClient::Client.new(compression: false, reconnect: true, reconnect_attempts: 5, reconnect_interval: 1)
24
+ count = 0
25
+ response = client.get("http://localhost:3000/outbounds/consumer/1") { |line|
26
+ logger.debug "line received: #{line}"
27
+ count = count + 1
28
+ client.interrupt if count > 20
29
+ }
30
+ end
31
+
32
+ end
data/spec/spec_helper.rb CHANGED
@@ -21,6 +21,8 @@ RSpec.configure do |config|
21
21
  HttpStreamingClient.logger.level = Logger::DEBUG
22
22
  HttpStreamingClient.logger.logfile = true
23
23
 
24
+ config.filter_run_excluding disabled: true
25
+
24
26
  end
25
27
 
26
28
  def logger
@@ -15,8 +15,8 @@ authorization = HttpStreamingClient::Oauth::Adobe.generate_authorization(url, US
15
15
  puts "#{TOKENAPIHOST}:access token: #{authorization}"
16
16
  client = HttpStreamingClient::Client.new(compression: false)
17
17
 
18
- NUM_RECORDS_PER_BATCH = 1000
19
- MAX_RECORDS = 50000
18
+ NUM_RECORDS_PER_BATCH = 5000
19
+ MAX_RECORDS = 3600000
20
20
 
21
21
  count = 0
22
22
  totalSize = 0
@@ -25,6 +25,7 @@ startTime = nil
25
25
  lastTime = nil
26
26
 
27
27
  puts "starting performance test run: #{Time.new.to_s}"
28
+ puts "stream: #{STREAMURL}"
28
29
 
29
30
  startTime = lastTime = Time.new
30
31
 
@@ -69,6 +70,7 @@ response = client.get(STREAMURL, {:headers => {'Authorization' => "Bearer #{auth
69
70
  stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
70
71
 
71
72
  puts stats.to_json
73
+ STDOUT.flush
72
74
 
73
75
  lastTime = now
74
76
  intervalSize = 0
@@ -24,6 +24,7 @@ startTime = nil
24
24
  lastTime = nil
25
25
 
26
26
  puts "starting performance test run: #{Time.new.to_s}"
27
+ puts "stream: #{url}"
27
28
 
28
29
  startTime = lastTime = Time.new
29
30
 
@@ -64,6 +65,7 @@ response = client.get(url, {:headers => {'Authorization' => "#{authorization}" }
64
65
  stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
65
66
 
66
67
  puts stats.to_json
68
+ STDOUT.flush
67
69
 
68
70
  lastTime = now
69
71
  intervalSize = 0
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: http_streaming_client
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.6
4
+ version: 0.8.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - David Tompkins
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-03 00:00:00.000000000 Z
11
+ date: 2014-01-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -128,6 +128,7 @@ files:
128
128
  - spec/adobe_spec.rb
129
129
  - spec/client_spec.rb
130
130
  - spec/oauth_spec.rb
131
+ - spec/reconnect_spec.rb
131
132
  - spec/spec_helper.rb
132
133
  - spec/twitter_spec.rb
133
134
  - tools/adobe_firehose.rb
@@ -164,5 +165,6 @@ test_files:
164
165
  - spec/adobe_spec.rb
165
166
  - spec/client_spec.rb
166
167
  - spec/oauth_spec.rb
168
+ - spec/reconnect_spec.rb
167
169
  - spec/spec_helper.rb
168
170
  - spec/twitter_spec.rb