http_streaming_client 0.8.6 → 0.8.8

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 607b852759a7216b85fa6d820d2bc70a2d50b475
4
- data.tar.gz: 49b554701c78b290e3b7e337a1b8fbcc12f0f726
3
+ metadata.gz: 8308d062ebb595dddc19e1ce5232829fb92bd9ce
4
+ data.tar.gz: cfbe2c60d0b7b021cbe09d80bee4ef8114115039
5
5
  SHA512:
6
- metadata.gz: c1d07b0e483dc0e81d83800d05d6170d28f00dad305f6e6f3644ffe96b9bac00ff203a7bd34fadf697245b10520cdb1e303b79442dc0bd6bd3a477bb52b99f63
7
- data.tar.gz: a26cf37f5791d4701ca0f0bf74cdedb009fb1a6d1ee58934f15dfad8a12ffb8c3755a3854c95b496fc2ca22bae69c70e8a04d40c7ada2efd54f41c5ead6ca2d2
6
+ metadata.gz: 50ee772c4e19aec32f3a771b1686b8beadd9d7fb2038cf08b0b65792f7b070f26b1bf74519c4f85deea9462e2a93df0939888d3c1233a383a6b21286ce93a6c1
7
+ data.tar.gz: 323dec68b2c4d8730d4c9803358f142e721efc3aa2b08745625eacd4ea6b59351e2394a24ca739d5baa5ef64f1343f2060d538a7d3bc2c7f3b3fdded0f17d066
data/CHANGELOG CHANGED
@@ -1,3 +1,11 @@
1
+ http_streaming_client 0.8.8 (1.7.2014)
2
+ ========================================
3
+ * Fixed Issue #1, memory leak due to accumulated messages within the buffer pipeline
4
+ * Added automatic reconnection support and configuration options
5
+ * Added automatic HTTP 301 redirect support
6
+ * Added new unit tests for reconnect and for memory leak detection
7
+ * Updated performance tests
8
+
1
9
  http_streaming_client 0.8.5 (12.10.2013)
2
10
  ========================================
3
11
  * Updates to support MRI ruby 2.0.0-p353
data/README.md CHANGED
@@ -9,39 +9,69 @@ Ruby HTTP client with support for HTTP 1.1 streaming, GZIP and zlib compressed s
9
9
 
10
10
  MRI ruby-2.0.0-p353. If you need it, install via rvm: https://rvm.io/
11
11
 
12
- ## Installation (local gem bundle/install)
12
+ ## Installation
13
13
 
14
- Execute the following to bundle the gem:
14
+ Add this line to your application's Gemfile:
15
15
 
16
- $ gem build http_streaming_client.gemspec
16
+ gem 'http_streaming_client'
17
17
 
18
- Then install the gem with:
18
+ And then execute:
19
+
20
+ $ bundle install
21
+
22
+ Or install it directly via gem with:
19
23
 
20
24
  $ gem install http_streaming_client
21
25
 
22
- ## Installation (via Github)
26
+ ## Simple Example
23
27
 
24
- Add this line to your application's Gemfile:
28
+ Twitter Firehose Sample Stream:
25
29
 
26
- gem 'http_streaming_client', :git => 'git@github.com:adobe-research/http_streaming_client.git'
30
+ ```ruby
31
+ require 'http_streaming_client'
27
32
 
28
- And then execute:
33
+ twitter_stream_url = "https://stream.twitter.com/1.1/statuses/sample.json"
29
34
 
30
- $ bundle install
35
+ # Generate the HMAC-SHA1 Twitter OAuth authorization header
36
+ authorization = HttpStreamingClient::Oauth::Twitter.generate_authorization(twitter_stream_url, "get", {}, OAUTH_CONSUMER_KEY, OAUTH_CONSUMER_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
31
37
 
32
- ## Installation (via rubygems.org)
38
+ # Configure the client
39
+ client = HttpStreamingClient::Client.new(compression: true, reconnect: true, reconnect_interval: 10, reconnect_attempts: 60)
33
40
 
34
- Add this line to your application's Gemfile:
41
+ # Open the connection and start processing messages
42
+ response = client.get(twitter_stream_url, {:headers => {'Authorization' => "#{authorization}" }}) { |line|
43
+ logger.info "Received a line that we could parse into JSON if we want: #{line}"
44
+ client.interrupt if we_want_to_stop_receiving_messages?
45
+ }
46
+ ```
35
47
 
36
- gem 'http_streaming_client'
48
+ For more examples, take a look at
37
49
 
38
- And then execute:
50
+ * spec/client_spec.rb
51
+ * tools/adobe_firehose.rb
52
+ * tools/adobe_firehose_performance_test.rb
53
+ * tools/twitter_firehose.rb
54
+ * tools/twitter_firehose_performance_test.rb
39
55
 
40
- $ bundle
56
+ ## Client Configuration Options
41
57
 
42
- Or install it yourself as:
58
+ The following options are supported as hash option parameters for HttpStreamingClient::Client.new:
43
59
 
44
- $ gem install http_streaming_client
60
+ GZip Compression
61
+
62
+ compression: true/false (default: true)
63
+
64
+ Automatic Socket Reconnect Functions
65
+
66
+ reconnect: true/false (default: false)
67
+
68
+ Reconnect Interval
69
+
70
+ reconnect_interval: interval_seconds (default: 1 second)
71
+
72
+ Maximum Reconnect Attempts
73
+
74
+ reconnect_attempts: num_attempts (default: 10)
45
75
 
46
76
  ## Logging
47
77
 
@@ -110,21 +140,6 @@ Individual test suites in the spec directory can be run via:
110
140
 
111
141
  An HTML coverage report is generated at the end of a full test run in the coverage directory.
112
142
 
113
- ## Examples
114
-
115
- Take a look at
116
-
117
- * spec/client_spec.rb
118
- * tools/adobe_firehose.rb
119
- * tools/adobe_firehose_performance_test.rb
120
- * tools/twitter_firehose.rb
121
- * tools/twitter_firehose_performance_test.rb
122
-
123
- ## TODO
124
-
125
- * connection management with reconnect functions
126
- * load/memory testing
127
-
128
143
  ## Fixed Issues
129
144
 
130
145
  * See [![CHANGELOG](CHANGELOG)](CHANGELOG)
@@ -40,10 +40,13 @@ module HttpStreamingClient
40
40
 
41
41
  class Client
42
42
 
43
- attr_accessor :socket, :interrupted, :compression_requested
43
+ attr_accessor :socket, :interrupted, :compression_requested, :reconnect_requested, :reconnect_attempts, :reconnect_interval
44
44
 
45
45
  ALLOWED_MIME_TYPES = ["application/json", "text/plain", "text/html"]
46
46
 
47
+ MAX_RECONNECT_ATTEMPTS = 10
48
+ RECONNECT_INTERVAL_SEC = 1
49
+
47
50
  def self.logger
48
51
  HttpStreamingClient.logger
49
52
  end
@@ -56,8 +59,24 @@ module HttpStreamingClient
56
59
  logger.debug("Client.new: #{opts}")
57
60
  @socket = nil
58
61
  @interrupted = false
62
+
59
63
  @compression_requested = opts[:compression].nil? ? true : opts[:compression]
60
64
  logger.debug("compression is #{@compression_requested}")
65
+
66
+ @reconnect_requested = opts[:reconnect].nil? ? false : opts[:reconnect]
67
+ logger.debug("reconnect is #{@reconnect_requested}")
68
+
69
+ if @reconnect_requested then
70
+ @reconnect_count = 0
71
+
72
+ @reconnect_attempts = opts[:reconnect_attempts].nil? ? MAX_RECONNECT_ATTEMPTS : opts[:reconnect_attempts]
73
+ logger.debug("reconnect_attempts is #{@reconnect_attempts}")
74
+
75
+ @reconnect_interval = opts[:reconnect_interval].nil? ? RECONNECT_INTERVAL_SEC : opts[:reconnect_interval]
76
+ logger.debug("reconnect_interval is #{@reconnect_interval}")
77
+
78
+ end
79
+
61
80
  end
62
81
 
63
82
  def self.get(uri, opts = {}, &block)
@@ -132,199 +151,226 @@ module HttpStreamingClient
132
151
  headers = default_headers.merge(opts[:headers] || {})
133
152
  logger.debug "request headers: #{headers}"
134
153
 
135
- socket = initialize_socket(uri, opts)
136
- request = "#{method} #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
137
- request << "Host: #{uri.host}\r\n"
138
- headers.each do |k, v|
139
- request << "#{k}: #{v}\r\n"
140
- end
141
- request << "\r\n"
142
- if method == "POST"
143
- request << body
144
- end
154
+ begin
145
155
 
146
- socket.write(request)
156
+ socket = initialize_socket(uri, opts)
147
157
 
148
- response_head = {}
149
- response_head[:headers] = {}
158
+ @reconnect_count = 0 if @reconnect_requested
150
159
 
151
- socket.each_line do |line|
152
- if line == "\r\n" then
153
- break
154
- else
155
- header = line.split(": ")
156
- if header.size == 1
157
- header = header[0].split(" ")
158
- response_head[:version] = header[0]
159
- response_head[:code] = header[1].to_i
160
- response_head[:msg] = header[2]
161
- logger.debug "HTTP response code is #{response_head[:code]}"
162
- else
163
- response_head[:headers][camelize_header_name(header[0])] = header[1].strip
164
- end
160
+ request = "#{method} #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
161
+ request << "Host: #{uri.host}\r\n"
162
+ headers.each do |k, v|
163
+ request << "#{k}: #{v}\r\n"
165
164
  end
166
- end
167
-
168
- logger.debug "response headers:#{response_head[:headers]}"
169
-
170
- if response_head[:code] == 301 then
171
- location = response_head[:headers]["Location"]
172
- raise InvalidRedirect, "Unable to find Location header for HTTP 301 response" if location.nil?
173
- logger.debug "Received HTTP 301 redirect to #{location}, following..."
174
- socket.close if !socket.nil? and !socket.closed?
175
- opts.delete(:socket)
176
- return request(method, location, opts, &block)
177
- end
178
-
179
- content_length = response_head[:headers]["Content-Length"].to_i
180
- logger.debug "content-length: #{content_length}"
181
-
182
- content_type = response_head[:headers]["Content-Type"].split(';').first
183
- logger.debug "content-type: #{content_type}"
184
-
185
- response_compression = false
186
-
187
- if ALLOWED_MIME_TYPES.include?(content_type)
188
- case response_head[:headers]["Content-Encoding"]
189
- when "gzip"
190
- response_compression = true
165
+ request << "\r\n"
166
+ if method == "POST"
167
+ request << body
191
168
  end
192
- else
193
- raise InvalidContentType, "invalid response MIME type: #{content_type}"
194
- end
195
169
 
196
- if (response_head[:code] != 200)
197
- s = "Received HTTP #{response_head[:code]} response"
198
- logger.debug "request: #{request}"
199
- response = socket.read(content_length)
200
- logger.debug "response: #{response}"
201
- raise HttpError.new(response_head[:code], "Received HTTP #{response_head[:code]} response", response_head[:headers])
202
- end
170
+ socket.write(request)
171
+
172
+ response_head = {}
173
+ response_head[:headers] = {}
203
174
 
204
- if response_head[:headers]["Transfer-Encoding"] == 'chunked'
205
- partial = nil
206
- decoder = nil
207
- response = ""
208
-
209
- if response_compression then
210
- logger.debug "response compression detected"
211
- if block_given? then
212
- decoder = HttpStreamingClient::Decoders::GZip.new { |line|
213
- logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
214
- block.call(line) unless @interrupted }
175
+ socket.each_line do |line|
176
+ if line == "\r\n" then
177
+ break
215
178
  else
216
- decoder = HttpStreamingClient::Decoders::GZip.new { |line|
217
- logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
218
- response << line unless @interrupted }
179
+ header = line.split(": ")
180
+ if header.size == 1
181
+ header = header[0].split(" ")
182
+ response_head[:version] = header[0]
183
+ response_head[:code] = header[1].to_i
184
+ response_head[:msg] = header[2]
185
+ logger.debug "HTTP response code is #{response_head[:code]}"
186
+ else
187
+ response_head[:headers][camelize_header_name(header[0])] = header[1].strip
188
+ end
219
189
  end
220
190
  end
221
191
 
222
- while !socket.eof? && (line = socket.gets)
223
- chunkLeft = 0
192
+ logger.debug "response headers:#{response_head[:headers]}"
224
193
 
225
- if line.match /^0\r\n/ then
226
- logger.debug "received zero length chunk, chunked encoding EOF"
227
- logger.debug "EOF line: #{line}"
228
- break
229
- end
194
+ if response_head[:code] == 301 then
195
+ location = response_head[:headers]["Location"]
196
+ raise InvalidRedirect, "Unable to find Location header for HTTP 301 response" if location.nil?
197
+ logger.debug "Received HTTP 301 redirect to #{location}, following..."
198
+ socket.close if !socket.nil? and !socket.closed?
199
+ opts.delete(:socket)
200
+ return request(method, location, opts, &block)
201
+ end
230
202
 
231
- next if line == "\r\n"
203
+ content_length = response_head[:headers]["Content-Length"].to_i
204
+ logger.debug "content-length: #{content_length}"
232
205
 
233
- size = line.hex
234
- logger.debug "chunk size:#{size}"
206
+ content_type = response_head[:headers]["Content-Type"].split(';').first
207
+ logger.debug "content-type: #{content_type}"
235
208
 
236
- partial = socket.read(size)
237
- next if partial.nil?
209
+ response_compression = false
238
210
 
239
- remaining = size-partial.size
240
- logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
241
- until remaining == 0
242
- partial << socket.read(remaining)
243
- remaining = size-partial.size
244
- logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
211
+ if ALLOWED_MIME_TYPES.include?(content_type)
212
+ case response_head[:headers]["Content-Encoding"]
213
+ when "gzip"
214
+ response_compression = true
245
215
  end
216
+ else
217
+ raise InvalidContentType, "invalid response MIME type: #{content_type}"
218
+ end
219
+
220
+ if (response_head[:code] != 200)
221
+ s = "Received HTTP #{response_head[:code]} response"
222
+ logger.debug "request: #{request}"
223
+ response = socket.read(content_length)
224
+ logger.debug "response: #{response}"
225
+ raise HttpError.new(response_head[:code], "Received HTTP #{response_head[:code]} response", response_head[:headers])
226
+ end
246
227
 
247
- return if @interrupted
228
+ if response_head[:headers]["Transfer-Encoding"] == 'chunked'
229
+ partial = nil
230
+ decoder = nil
231
+ response = ""
248
232
 
249
233
  if response_compression then
250
- decoder << partial
251
- else
234
+ logger.debug "response compression detected"
252
235
  if block_given? then
253
- yield partial
236
+ decoder = HttpStreamingClient::Decoders::GZip.new { |line|
237
+ logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
238
+ block.call(line) unless @interrupted }
254
239
  else
255
- logger.debug "no block specified, returning chunk results and halting streaming response"
256
- response << partial
240
+ decoder = HttpStreamingClient::Decoders::GZip.new { |line|
241
+ logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
242
+ response << line unless @interrupted }
257
243
  end
258
244
  end
259
- end
260
245
 
261
- return response
262
-
263
- else
264
- # Not chunked transfer encoding, but potentially gzip'd, and potentially streaming with content-length = 0
265
-
266
- if content_length > 0 then
267
- bits = socket.read(content_length)
268
- logger.debug "read #{content_length} bytes"
269
- return bits if !response_compression
270
- logger.debug "response compression detected"
271
- begin
272
- decoder = Zlib::GzipReader.new(StringIO.new(bits))
273
- return decoder.read
274
- rescue Zlib::Error
275
- raise DecoderError
276
- end
277
- end
246
+ while !socket.eof? && (line = socket.gets)
247
+ chunkLeft = 0
278
248
 
279
- if response_compression then
249
+ if line.match /^0\r\n/ then
250
+ logger.debug "received zero length chunk, chunked encoding EOF"
251
+ logger.debug "EOF line: #{line}"
252
+ break
253
+ end
280
254
 
281
- logger.debug "response compression detected"
282
- decoder = nil
283
- response = ""
255
+ next if line == "\r\n"
284
256
 
285
- if block_given? then
286
- decoder = HttpStreamingClient::Decoders::GZip.new { |line|
287
- logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
288
- block.call(line) unless @interrupted }
289
- else
290
- decoder = HttpStreamingClient::Decoders::GZip.new { |line|
291
- logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
292
- response << line unless @interrupted }
293
- end
257
+ size = line.hex
258
+ logger.debug "chunk size:#{size}"
259
+
260
+ partial = socket.read(size)
261
+ next if partial.nil?
262
+
263
+ remaining = size-partial.size
264
+ logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
265
+ until remaining == 0
266
+ partial << socket.read(remaining)
267
+ remaining = size-partial.size
268
+ logger.debug "read #{partial.size} bytes, #{remaining} bytes remaining"
269
+ end
270
+
271
+ return if @interrupted
294
272
 
295
- while (!socket.eof? and !(line = socket.read_nonblock(2048)).nil?)
296
- logger.debug "read compressed line, #{line.size} bytes"
297
- decoder << line
298
- break response if @interrupted
273
+ if response_compression then
274
+ decoder << partial
275
+ else
276
+ if block_given? then
277
+ yield partial
278
+ else
279
+ logger.debug "no block specified, returning chunk results and halting streaming response"
280
+ response << partial
281
+ interrupt
282
+ end
283
+ end
299
284
  end
300
- logger.debug "EOF detected"
301
- decoder = nil
302
285
 
286
+ logger.debug "socket EOF detected" if socket.eof?
287
+ raise ReconnectRequest if @reconnect_requested
303
288
  return response
304
289
 
305
290
  else
291
+ # Not chunked transfer encoding, but potentially gzip'd, and potentially streaming with content-length = 0
292
+
293
+ if content_length > 0 then
294
+ bits = socket.read(content_length)
295
+ logger.debug "read #{content_length} bytes"
296
+ return bits if !response_compression
297
+ logger.debug "response compression detected"
298
+ begin
299
+ decoder = Zlib::GzipReader.new(StringIO.new(bits))
300
+ return decoder.read
301
+ rescue Zlib::Error
302
+ raise DecoderError
303
+ end
304
+ end
306
305
 
307
- response = ""
306
+ if response_compression then
307
+
308
+ logger.debug "response compression detected"
309
+ decoder = nil
310
+ response = ""
308
311
 
309
- while (!socket.eof? and !(line = socket.readline).nil?)
310
312
  if block_given? then
311
- yield line
312
- logger.debug "read #{line.size} bytes"
313
+ decoder = HttpStreamingClient::Decoders::GZip.new { |line|
314
+ logger.debug "read #{line.size} uncompressed bytes, decoder queue bytes:#{decoder.size}"
315
+ block.call(line) unless @interrupted }
313
316
  else
314
- logger.debug "read #{line.size} bytes, #{response.size} bytes total"
315
- response << line
317
+ decoder = HttpStreamingClient::Decoders::GZip.new { |line|
318
+ logger.debug "read #{line.size} uncompressed bytes, #{response.size} bytes total, decoder queue bytes:#{decoder.size}"
319
+ response << line unless @interrupted }
316
320
  end
317
- break if @interrupted
318
- end
319
321
 
320
- return response
322
+ while (!socket.eof? and !(line = socket.read_nonblock(2048)).nil?)
323
+ logger.debug "read compressed line, #{line.size} bytes"
324
+ decoder << line
325
+ break response if @interrupted
326
+ end
327
+
328
+ logger.debug "EOF detected"
329
+ raise ReconnectRequest if @reconnect_requested
330
+ return response
331
+
332
+ else
333
+
334
+ response = ""
335
+
336
+ while (!socket.eof? and !(line = socket.readline).nil?)
337
+ if block_given? then
338
+ yield line
339
+ logger.debug "read #{line.size} bytes"
340
+ else
341
+ logger.debug "read #{line.size} bytes, #{response.size} bytes total"
342
+ response << line
343
+ end
344
+ break if @interrupted
345
+ end
346
+
347
+ raise ReconnectRequest if @reconnect_requested
348
+ return response
349
+
350
+ end
351
+ end
352
+ rescue => e
353
+ return if @interrupted
354
+ logger.debug "Error Detected: #{e}" unless e.instance_of? ReconnectRequest
355
+ decoder.close if !decoder.nil?
356
+ socket.close if !socket.nil? and !socket.closed?
357
+ opts.delete(:socket)
321
358
 
359
+ if @reconnect_requested then
360
+ logger.info "Connection closed. Reconnect requested. Trying..."
361
+ @reconnect_count = @reconnect_count + 1
362
+ sleep @reconnect_interval
363
+ retry if @reconnect_count < @reconnect_attempts
364
+ logger.info "Maximum number of failed reconnect attempts reached (#{@reconnect_attempts}). Exiting."
322
365
  end
366
+
367
+ raise e unless e.instance_of? ReconnectRequest
323
368
  end
324
369
  ensure
325
370
  logger.debug "ensure socket closed"
326
371
  decoder.close if !decoder.nil?
327
372
  socket.close if !socket.nil? and !socket.closed?
373
+ opts.delete(:socket)
328
374
  end
329
375
 
330
376
  private
@@ -340,6 +386,8 @@ module HttpStreamingClient
340
386
  uri = URI.parse(uri)
341
387
  end
342
388
 
389
+ logger.debug "Connecting to #{uri.host}:#{uri.port}..."
390
+
343
391
  @socket = TCPSocket.new(uri.host, uri.port)
344
392
 
345
393
  if uri.scheme.eql? "https"
@@ -351,6 +399,7 @@ module HttpStreamingClient
351
399
  end
352
400
  end
353
401
 
402
+ logger.debug "Connected to #{uri.host}:#{uri.port}"
354
403
  opts.merge!({:socket => @socket})
355
404
  @interrupted = false
356
405
  return opts[:socket]
@@ -105,7 +105,7 @@ module HttpStreamingClient
105
105
 
106
106
  protected
107
107
 
108
- class GZipBufferIO
108
+ class GZipBufferIO < IO
109
109
 
110
110
  def logger
111
111
  HttpStreamingClient.logger
@@ -29,9 +29,12 @@
29
29
 
30
30
  module HttpStreamingClient
31
31
 
32
+
32
33
  class InvalidContentType < Exception; end
33
34
 
34
35
  class InvalidRedirect < Exception; end
36
+
37
+ class ReconnectRequest < StandardError; end
35
38
 
36
39
  class HttpTimeOut < StandardError; end
37
40
 
@@ -28,5 +28,5 @@
28
28
  ###########################################################################
29
29
 
30
30
  module HttpStreamingClient
31
- VERSION = "0.8.6"
31
+ VERSION = "0.8.8"
32
32
  end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper'
2
+
3
+
4
+ describe HttpStreamingClient do
5
+
6
+ # currently disabled, requires a server that can be killed to simulate dropped connections
7
+
8
+ it "should receive exactly 10 messages, no reconnect" do
9
+ #it "should receive exactly 10 messages, no reconnect", :disabled => true do
10
+
11
+ count = 0
12
+ client = HttpStreamingClient::Client.new(compression: false)
13
+ response = client.get("http://localhost:3000/outbounds/consumer/1") { |line|
14
+ logger.debug "line received: #{line}"
15
+ count = count + 1
16
+ }
17
+ expect(response).to be_true
18
+ expect(count).to be(10)
19
+ end
20
+
21
+ it "should reconnect on any error or EOF" do
22
+
23
+ client = HttpStreamingClient::Client.new(compression: false, reconnect: true, reconnect_attempts: 5, reconnect_interval: 1)
24
+ count = 0
25
+ response = client.get("http://localhost:3000/outbounds/consumer/1") { |line|
26
+ logger.debug "line received: #{line}"
27
+ count = count + 1
28
+ client.interrupt if count > 20
29
+ }
30
+ end
31
+
32
+ end
data/spec/spec_helper.rb CHANGED
@@ -21,6 +21,8 @@ RSpec.configure do |config|
21
21
  HttpStreamingClient.logger.level = Logger::DEBUG
22
22
  HttpStreamingClient.logger.logfile = true
23
23
 
24
+ config.filter_run_excluding disabled: true
25
+
24
26
  end
25
27
 
26
28
  def logger
@@ -15,8 +15,8 @@ authorization = HttpStreamingClient::Oauth::Adobe.generate_authorization(url, US
15
15
  puts "#{TOKENAPIHOST}:access token: #{authorization}"
16
16
  client = HttpStreamingClient::Client.new(compression: false)
17
17
 
18
- NUM_RECORDS_PER_BATCH = 1000
19
- MAX_RECORDS = 50000
18
+ NUM_RECORDS_PER_BATCH = 5000
19
+ MAX_RECORDS = 3600000
20
20
 
21
21
  count = 0
22
22
  totalSize = 0
@@ -25,6 +25,7 @@ startTime = nil
25
25
  lastTime = nil
26
26
 
27
27
  puts "starting performance test run: #{Time.new.to_s}"
28
+ puts "stream: #{STREAMURL}"
28
29
 
29
30
  startTime = lastTime = Time.new
30
31
 
@@ -69,6 +70,7 @@ response = client.get(STREAMURL, {:headers => {'Authorization' => "Bearer #{auth
69
70
  stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
70
71
 
71
72
  puts stats.to_json
73
+ STDOUT.flush
72
74
 
73
75
  lastTime = now
74
76
  intervalSize = 0
@@ -24,6 +24,7 @@ startTime = nil
24
24
  lastTime = nil
25
25
 
26
26
  puts "starting performance test run: #{Time.new.to_s}"
27
+ puts "stream: #{url}"
27
28
 
28
29
  startTime = lastTime = Time.new
29
30
 
@@ -64,6 +65,7 @@ response = client.get(url, {:headers => {'Authorization' => "#{authorization}" }
64
65
  stats['interval_kbytes_per_sec'] = (intervalSize / intervalElapsedTime / 1024).round(2).to_s
65
66
 
66
67
  puts stats.to_json
68
+ STDOUT.flush
67
69
 
68
70
  lastTime = now
69
71
  intervalSize = 0
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: http_streaming_client
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.6
4
+ version: 0.8.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - David Tompkins
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-03 00:00:00.000000000 Z
11
+ date: 2014-01-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -128,6 +128,7 @@ files:
128
128
  - spec/adobe_spec.rb
129
129
  - spec/client_spec.rb
130
130
  - spec/oauth_spec.rb
131
+ - spec/reconnect_spec.rb
131
132
  - spec/spec_helper.rb
132
133
  - spec/twitter_spec.rb
133
134
  - tools/adobe_firehose.rb
@@ -164,5 +165,6 @@ test_files:
164
165
  - spec/adobe_spec.rb
165
166
  - spec/client_spec.rb
166
167
  - spec/oauth_spec.rb
168
+ - spec/reconnect_spec.rb
167
169
  - spec/spec_helper.rb
168
170
  - spec/twitter_spec.rb