twitter-stream 0.1.13 → 0.1.14
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of twitter-stream might be problematic. Click here for more details.
- data/VERSION +1 -1
- data/fixtures/twitter/{basic_http.txt → tweets.txt} +4 -14
- data/lib/twitter/json_stream.rb +80 -65
- data/spec/spec_helper.rb +25 -0
- data/spec/twitter/json_stream_spec.rb +82 -5
- data/twitter-stream.gemspec +3 -2
- metadata +28 -10
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.14
|
@@ -1,14 +1,4 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
{"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world! Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}
|
7
|
-
|
8
|
-
{"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch: http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}
|
9
|
-
|
10
|
-
{"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats. http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
{"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}
|
1
|
+
{"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world! Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}
|
2
|
+
{"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch: http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}
|
3
|
+
{"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats. http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}
|
4
|
+
{"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}
|
data/lib/twitter/json_stream.rb
CHANGED
@@ -2,6 +2,7 @@ require 'eventmachine'
|
|
2
2
|
require 'em/buftok'
|
3
3
|
require 'uri'
|
4
4
|
require 'simple_oauth'
|
5
|
+
require 'http/parser'
|
5
6
|
|
6
7
|
module Twitter
|
7
8
|
class JSONStream < EventMachine::Connection
|
@@ -20,21 +21,22 @@ module Twitter
|
|
20
21
|
RETRIES_MAX = 10
|
21
22
|
|
22
23
|
DEFAULT_OPTIONS = {
|
23
|
-
:method
|
24
|
-
:path
|
25
|
-
:content_type
|
26
|
-
:content
|
27
|
-
:path
|
28
|
-
:host
|
29
|
-
:port
|
30
|
-
:ssl
|
31
|
-
:user_agent
|
32
|
-
:timeout
|
33
|
-
:proxy
|
34
|
-
:auth
|
35
|
-
:oauth
|
36
|
-
:filters
|
37
|
-
:params
|
24
|
+
:method => 'GET',
|
25
|
+
:path => '/',
|
26
|
+
:content_type => "application/x-www-form-urlencoded",
|
27
|
+
:content => '',
|
28
|
+
:path => '/1/statuses/filter.json',
|
29
|
+
:host => 'stream.twitter.com',
|
30
|
+
:port => 80,
|
31
|
+
:ssl => false,
|
32
|
+
:user_agent => 'TwitterStream',
|
33
|
+
:timeout => 0,
|
34
|
+
:proxy => ENV['HTTP_PROXY'],
|
35
|
+
:auth => nil,
|
36
|
+
:oauth => {},
|
37
|
+
:filters => [],
|
38
|
+
:params => {},
|
39
|
+
:auto_reconnect => true
|
38
40
|
}
|
39
41
|
|
40
42
|
attr_accessor :code
|
@@ -89,6 +91,10 @@ module Twitter
|
|
89
91
|
@max_reconnects_callback = block
|
90
92
|
end
|
91
93
|
|
94
|
+
def on_close &block
|
95
|
+
@close_callback = block
|
96
|
+
end
|
97
|
+
|
92
98
|
def stop
|
93
99
|
@gracefully_closed = true
|
94
100
|
close_connection
|
@@ -101,20 +107,18 @@ module Twitter
|
|
101
107
|
end
|
102
108
|
|
103
109
|
def unbind
|
104
|
-
|
105
|
-
|
110
|
+
if @state == :stream && !@buffer.empty?
|
111
|
+
parse_stream_line(@buffer.flush)
|
112
|
+
end
|
113
|
+
schedule_reconnect if @options[:auto_reconnect] && !@gracefully_closed
|
114
|
+
@close_callback.call if @close_callback
|
115
|
+
|
106
116
|
end
|
107
117
|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
end
|
113
|
-
rescue Exception => e
|
114
|
-
receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
|
115
|
-
close_connection
|
116
|
-
return
|
117
|
-
end
|
118
|
+
# Receives raw data from the HTTP connection and pushes it into the
|
119
|
+
# HTTP parser which then drives subsequent callbacks.
|
120
|
+
def receive_data(data)
|
121
|
+
@parser << data
|
118
122
|
end
|
119
123
|
|
120
124
|
def connection_completed
|
@@ -175,9 +179,40 @@ module Twitter
|
|
175
179
|
def reset_state
|
176
180
|
set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
|
177
181
|
@code = 0
|
178
|
-
@headers =
|
182
|
+
@headers = {}
|
179
183
|
@state = :init
|
180
184
|
@buffer = BufferedTokenizer.new("\r", MAX_LINE_LENGTH)
|
185
|
+
@stream = ''
|
186
|
+
|
187
|
+
@parser = Http::Parser.new
|
188
|
+
@parser.on_headers_complete = method(:handle_headers_complete)
|
189
|
+
@parser.on_body = method(:receive_stream_data)
|
190
|
+
end
|
191
|
+
|
192
|
+
# Called when the status line and all headers have been read from the
|
193
|
+
# stream.
|
194
|
+
def handle_headers_complete(headers)
|
195
|
+
@code = @parser.status_code.to_i
|
196
|
+
if @code != 200
|
197
|
+
receive_error("invalid status code: #{@code}.")
|
198
|
+
end
|
199
|
+
self.headers = headers
|
200
|
+
@state = :stream
|
201
|
+
end
|
202
|
+
|
203
|
+
# Called every time a chunk of data is read from the connection once it has
|
204
|
+
# been opened and after the headers have been processed.
|
205
|
+
def receive_stream_data(data)
|
206
|
+
begin
|
207
|
+
@buffer.extract(data).each do |line|
|
208
|
+
parse_stream_line(line)
|
209
|
+
end
|
210
|
+
@stream = ''
|
211
|
+
rescue Exception => e
|
212
|
+
receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
|
213
|
+
close_connection
|
214
|
+
return
|
215
|
+
end
|
181
216
|
end
|
182
217
|
|
183
218
|
def send_request
|
@@ -217,22 +252,18 @@ module Twitter
|
|
217
252
|
data << "Content-type: #{@options[:content_type]}"
|
218
253
|
data << "Content-length: #{content.length}"
|
219
254
|
end
|
255
|
+
|
256
|
+
if @options[:headers]
|
257
|
+
@options[:headers].each do |name,value|
|
258
|
+
data << "#{name}: #{value}"
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
220
262
|
data << "\r\n"
|
221
263
|
|
222
264
|
send_data data.join("\r\n") << content
|
223
265
|
end
|
224
266
|
|
225
|
-
def receive_line ln
|
226
|
-
case @state
|
227
|
-
when :init
|
228
|
-
parse_response_line ln
|
229
|
-
when :headers
|
230
|
-
parse_header_line ln
|
231
|
-
when :stream
|
232
|
-
parse_stream_line ln
|
233
|
-
end
|
234
|
-
end
|
235
|
-
|
236
267
|
def receive_error e
|
237
268
|
@error_callback.call(e) if @error_callback
|
238
269
|
end
|
@@ -240,34 +271,18 @@ module Twitter
|
|
240
271
|
def parse_stream_line ln
|
241
272
|
ln.strip!
|
242
273
|
unless ln.empty?
|
243
|
-
if ln[0,1] == '{'
|
244
|
-
@
|
274
|
+
if ln[0,1] == '{' || ln[ln.length-1,1] == '}'
|
275
|
+
@stream << ln
|
276
|
+
if @stream[0,1] == '{' && @stream[@stream.length-1,1] == '}'
|
277
|
+
@each_item_callback.call(@stream) if @each_item_callback
|
278
|
+
@stream = ''
|
279
|
+
end
|
245
280
|
end
|
246
281
|
end
|
247
282
|
end
|
248
283
|
|
249
|
-
def parse_header_line ln
|
250
|
-
ln.strip!
|
251
|
-
if ln.empty?
|
252
|
-
reset_timeouts if @code == 200
|
253
|
-
@state = :stream
|
254
|
-
else
|
255
|
-
headers << ln
|
256
|
-
end
|
257
|
-
end
|
258
|
-
|
259
|
-
def parse_response_line ln
|
260
|
-
if ln =~ /\AHTTP\/1\.[01] ([\d]{3})/
|
261
|
-
@code = $1.to_i
|
262
|
-
@state = :headers
|
263
|
-
receive_error("invalid status code: #{@code}. #{ln}") unless @code == 200
|
264
|
-
else
|
265
|
-
receive_error('invalid response')
|
266
|
-
close_connection
|
267
|
-
end
|
268
|
-
end
|
269
|
-
|
270
284
|
def reset_timeouts
|
285
|
+
set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
|
271
286
|
@nf_last_reconnect = @af_last_reconnect = nil
|
272
287
|
@reconnect_retries = 0
|
273
288
|
end
|
@@ -309,15 +324,15 @@ module Twitter
|
|
309
324
|
def params
|
310
325
|
flat = {}
|
311
326
|
@options[:params].merge( :track => @options[:filters] ).each do |param, val|
|
312
|
-
next if val.empty?
|
327
|
+
next if val.to_s.empty? || (val.respond_to?(:empty?) && val.empty?)
|
313
328
|
val = val.join(",") if val.respond_to?(:join)
|
314
|
-
flat[
|
329
|
+
flat[param.to_s] = val.to_s
|
315
330
|
end
|
316
331
|
flat
|
317
332
|
end
|
318
333
|
|
319
334
|
def query
|
320
|
-
params.map{|
|
335
|
+
params.map{|param, value| [escape(param), escape(value)].join("=")}.sort.join("&")
|
321
336
|
end
|
322
337
|
|
323
338
|
def escape str
|
data/spec/spec_helper.rb
CHANGED
@@ -26,4 +26,29 @@ def connect_stream(opts={}, &blk)
|
|
26
26
|
blk.call if blk
|
27
27
|
EM.add_timer(stop_in){ EM.stop }
|
28
28
|
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def http_response(status_code, status_text, headers, body)
|
32
|
+
res = "HTTP/1.1 #{status_code} #{status_text}\r\n"
|
33
|
+
headers = {
|
34
|
+
"Content-Type"=>"application/json",
|
35
|
+
"Transfer-Encoding"=>"chunked"
|
36
|
+
}.merge(headers)
|
37
|
+
headers.each do |key,value|
|
38
|
+
res << "#{key}: #{value}\r\n"
|
39
|
+
end
|
40
|
+
res << "\r\n"
|
41
|
+
if headers["Transfer-Encoding"] == "chunked" && body.kind_of?(Array)
|
42
|
+
body.each do |data|
|
43
|
+
res << http_chunk(data)
|
44
|
+
end
|
45
|
+
else
|
46
|
+
res << body
|
47
|
+
end
|
48
|
+
res
|
49
|
+
end
|
50
|
+
|
51
|
+
def http_chunk(data)
|
52
|
+
# See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
|
53
|
+
"#{data.length.to_s(16)}\r\n#{data}\r\n"
|
29
54
|
end
|
@@ -74,7 +74,9 @@ describe JSONStream do
|
|
74
74
|
context "on valid stream" do
|
75
75
|
attr_reader :stream
|
76
76
|
before :each do
|
77
|
-
$
|
77
|
+
$body = File.readlines(fixture_path("twitter/tweets.txt"))
|
78
|
+
$body.each {|tweet| tweet.strip!; tweet << "\r" }
|
79
|
+
$data_to_send = http_response(200,"OK",{},$body)
|
78
80
|
$recieved_data = ''
|
79
81
|
$close_connection = false
|
80
82
|
end
|
@@ -92,13 +94,13 @@ describe JSONStream do
|
|
92
94
|
it "should parse headers" do
|
93
95
|
connect_stream
|
94
96
|
stream.code.should == 200
|
95
|
-
stream.headers
|
97
|
+
stream.headers.keys.map{|k| k.downcase}.should include('content-type')
|
96
98
|
end
|
97
99
|
|
98
100
|
it "should parse headers even after connection close" do
|
99
101
|
connect_stream
|
100
102
|
stream.code.should == 200
|
101
|
-
stream.headers
|
103
|
+
stream.headers.keys.map{|k| k.downcase}.should include('content-type')
|
102
104
|
end
|
103
105
|
|
104
106
|
it "should extract records" do
|
@@ -106,6 +108,26 @@ describe JSONStream do
|
|
106
108
|
$recieved_data.upcase.should include('USER-AGENT: TEST_USER_AGENT')
|
107
109
|
end
|
108
110
|
|
111
|
+
it 'should allow custom headers' do
|
112
|
+
connect_stream :headers => { 'From' => 'twitter-stream' }
|
113
|
+
$recieved_data.upcase.should include('FROM: TWITTER-STREAM')
|
114
|
+
end
|
115
|
+
|
116
|
+
it "should deliver each item" do
|
117
|
+
items = []
|
118
|
+
connect_stream do
|
119
|
+
stream.each_item do |item|
|
120
|
+
items << item
|
121
|
+
end
|
122
|
+
end
|
123
|
+
# Extract only the tweets from the fixture
|
124
|
+
tweets = $body.map{|l| l.strip }.select{|l| l =~ /^\{/ }
|
125
|
+
items.size.should == tweets.size
|
126
|
+
tweets.each_with_index do |tweet,i|
|
127
|
+
items[i].should == tweet
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
109
131
|
it "should send correct user agent" do
|
110
132
|
connect_stream
|
111
133
|
end
|
@@ -118,6 +140,12 @@ describe JSONStream do
|
|
118
140
|
end
|
119
141
|
end
|
120
142
|
|
143
|
+
it "should not reconnect on network failure when not configured to auto reconnect" do
|
144
|
+
connect_stream(:auto_reconnect => false) do
|
145
|
+
stream.should_receive(:reconnect).never
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
121
149
|
it "should reconnect with 0.25 at base" do
|
122
150
|
connect_stream do
|
123
151
|
stream.should_receive(:reconnect_after).with(0.25)
|
@@ -164,6 +192,12 @@ describe JSONStream do
|
|
164
192
|
end
|
165
193
|
end
|
166
194
|
|
195
|
+
it "should not reconnect on inactivity when not configured to auto reconnect" do
|
196
|
+
connect_stream(:stop_in => 1.5, :auto_reconnect => false) do
|
197
|
+
stream.should_receive(:reconnect).never
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
167
201
|
it_should_behave_like "network failure"
|
168
202
|
end
|
169
203
|
|
@@ -186,8 +220,8 @@ describe JSONStream do
|
|
186
220
|
context "on application failure" do
|
187
221
|
attr_reader :stream
|
188
222
|
before :each do
|
189
|
-
$data_to_send =
|
190
|
-
$close_connection =
|
223
|
+
$data_to_send = "HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Basic realm=\"Firehose\"\r\n\r\n"
|
224
|
+
$close_connection = false
|
191
225
|
end
|
192
226
|
|
193
227
|
it "should reconnect on application failure 10 at base" do
|
@@ -196,6 +230,12 @@ describe JSONStream do
|
|
196
230
|
end
|
197
231
|
end
|
198
232
|
|
233
|
+
it "should not reconnect on application failure 10 at base when not configured to auto reconnect" do
|
234
|
+
connect_stream(:auto_reconnect => false) do
|
235
|
+
stream.should_receive(:reconnect_after).never
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
199
239
|
it "should reconnect with exponential timeout" do
|
200
240
|
connect_stream do
|
201
241
|
stream.af_last_reconnect = 160
|
@@ -210,4 +250,41 @@ describe JSONStream do
|
|
210
250
|
end
|
211
251
|
end
|
212
252
|
end
|
253
|
+
|
254
|
+
context "on stream with chunked transfer encoding" do
|
255
|
+
attr_reader :stream
|
256
|
+
before :each do
|
257
|
+
$recieved_data = ''
|
258
|
+
$close_connection = false
|
259
|
+
end
|
260
|
+
|
261
|
+
it "should ignore empty lines" do
|
262
|
+
body_chunks = ["{\"screen"+"_name\"",":\"user1\"}\r\r\r{","\"id\":9876}\r\r"]
|
263
|
+
$data_to_send = http_response(200,"OK",{},body_chunks)
|
264
|
+
items = []
|
265
|
+
connect_stream do
|
266
|
+
stream.each_item do |item|
|
267
|
+
items << item
|
268
|
+
end
|
269
|
+
end
|
270
|
+
items.size.should == 2
|
271
|
+
items[0].should == '{"screen_name":"user1"}'
|
272
|
+
items[1].should == '{"id":9876}'
|
273
|
+
end
|
274
|
+
|
275
|
+
it "should parse full entities even if split" do
|
276
|
+
body_chunks = ["{\"id\"",":1234}\r{","\"id\":9876}"]
|
277
|
+
$data_to_send = http_response(200,"OK",{},body_chunks)
|
278
|
+
items = []
|
279
|
+
connect_stream do
|
280
|
+
stream.each_item do |item|
|
281
|
+
items << item
|
282
|
+
end
|
283
|
+
end
|
284
|
+
items.size.should == 2
|
285
|
+
items[0].should == '{"id":1234}'
|
286
|
+
items[1].should == '{"id":9876}'
|
287
|
+
end
|
288
|
+
end
|
289
|
+
|
213
290
|
end
|
data/twitter-stream.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = %q{twitter-stream}
|
5
|
-
s.version = "0.1.
|
5
|
+
s.version = "0.1.14"
|
6
6
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
8
|
s.authors = ["Vladimir Kolesnikov"]
|
@@ -19,8 +19,9 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.rdoc_options = ["--charset=UTF-8"]
|
20
20
|
s.extra_rdoc_files = ["README.markdown"]
|
21
21
|
|
22
|
-
s.add_runtime_dependency('eventmachine', "
|
22
|
+
s.add_runtime_dependency('eventmachine', ">= 0.12.8")
|
23
23
|
s.add_runtime_dependency('simple_oauth', '~> 0.1.4')
|
24
|
+
s.add_runtime_dependency('http_parser.rb', '~> 0.5.1')
|
24
25
|
s.add_development_dependency('rspec', "~> 2.5.0")
|
25
26
|
|
26
27
|
s.files = `git ls-files`.split("\n")
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: twitter-stream
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
5
|
-
prerelease:
|
4
|
+
hash: 7
|
5
|
+
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 1
|
9
|
-
-
|
10
|
-
version: 0.1.
|
9
|
+
- 14
|
10
|
+
version: 0.1.14
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Vladimir Kolesnikov
|
@@ -15,7 +15,8 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2010-10-05 00:00:00
|
18
|
+
date: 2010-10-05 00:00:00 -07:00
|
19
|
+
default_executable:
|
19
20
|
dependencies:
|
20
21
|
- !ruby/object:Gem::Dependency
|
21
22
|
name: eventmachine
|
@@ -23,7 +24,7 @@ dependencies:
|
|
23
24
|
requirement: &id001 !ruby/object:Gem::Requirement
|
24
25
|
none: false
|
25
26
|
requirements:
|
26
|
-
- -
|
27
|
+
- - ">="
|
27
28
|
- !ruby/object:Gem::Version
|
28
29
|
hash: 63
|
29
30
|
segments:
|
@@ -50,9 +51,25 @@ dependencies:
|
|
50
51
|
type: :runtime
|
51
52
|
version_requirements: *id002
|
52
53
|
- !ruby/object:Gem::Dependency
|
53
|
-
name:
|
54
|
+
name: http_parser.rb
|
54
55
|
prerelease: false
|
55
56
|
requirement: &id003 !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
hash: 9
|
62
|
+
segments:
|
63
|
+
- 0
|
64
|
+
- 5
|
65
|
+
- 1
|
66
|
+
version: 0.5.1
|
67
|
+
type: :runtime
|
68
|
+
version_requirements: *id003
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rspec
|
71
|
+
prerelease: false
|
72
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
56
73
|
none: false
|
57
74
|
requirements:
|
58
75
|
- - ~>
|
@@ -64,7 +81,7 @@ dependencies:
|
|
64
81
|
- 0
|
65
82
|
version: 2.5.0
|
66
83
|
type: :development
|
67
|
-
version_requirements: *
|
84
|
+
version_requirements: *id004
|
68
85
|
description: Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only.
|
69
86
|
email: voloko@gmail.com
|
70
87
|
executables: []
|
@@ -82,11 +99,12 @@ files:
|
|
82
99
|
- Rakefile
|
83
100
|
- VERSION
|
84
101
|
- examples/reader.rb
|
85
|
-
- fixtures/twitter/
|
102
|
+
- fixtures/twitter/tweets.txt
|
86
103
|
- lib/twitter/json_stream.rb
|
87
104
|
- spec/spec_helper.rb
|
88
105
|
- spec/twitter/json_stream_spec.rb
|
89
106
|
- twitter-stream.gemspec
|
107
|
+
has_rdoc: true
|
90
108
|
homepage: http://github.com/voloko/twitter-stream
|
91
109
|
licenses: []
|
92
110
|
|
@@ -118,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
118
136
|
requirements: []
|
119
137
|
|
120
138
|
rubyforge_project:
|
121
|
-
rubygems_version: 1.7
|
139
|
+
rubygems_version: 1.3.7
|
122
140
|
signing_key:
|
123
141
|
specification_version: 3
|
124
142
|
summary: Twitter realtime API client
|