twitter-stream 0.1.13 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/fixtures/twitter/{basic_http.txt → tweets.txt} +4 -14
- data/lib/twitter/json_stream.rb +80 -65
- data/spec/spec_helper.rb +25 -0
- data/spec/twitter/json_stream_spec.rb +82 -5
- data/twitter-stream.gemspec +3 -2
- metadata +28 -10
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.1.
|
|
1
|
+
0.1.14
|
|
@@ -1,14 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
{"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world! Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}
|
|
7
|
-
|
|
8
|
-
{"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch: http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}
|
|
9
|
-
|
|
10
|
-
{"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats. http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
{"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}
|
|
1
|
+
{"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world! Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}
|
|
2
|
+
{"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch: http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}
|
|
3
|
+
{"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats. http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}
|
|
4
|
+
{"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}
|
data/lib/twitter/json_stream.rb
CHANGED
|
@@ -2,6 +2,7 @@ require 'eventmachine'
|
|
|
2
2
|
require 'em/buftok'
|
|
3
3
|
require 'uri'
|
|
4
4
|
require 'simple_oauth'
|
|
5
|
+
require 'http/parser'
|
|
5
6
|
|
|
6
7
|
module Twitter
|
|
7
8
|
class JSONStream < EventMachine::Connection
|
|
@@ -20,21 +21,22 @@ module Twitter
|
|
|
20
21
|
RETRIES_MAX = 10
|
|
21
22
|
|
|
22
23
|
DEFAULT_OPTIONS = {
|
|
23
|
-
:method
|
|
24
|
-
:path
|
|
25
|
-
:content_type
|
|
26
|
-
:content
|
|
27
|
-
:path
|
|
28
|
-
:host
|
|
29
|
-
:port
|
|
30
|
-
:ssl
|
|
31
|
-
:user_agent
|
|
32
|
-
:timeout
|
|
33
|
-
:proxy
|
|
34
|
-
:auth
|
|
35
|
-
:oauth
|
|
36
|
-
:filters
|
|
37
|
-
:params
|
|
24
|
+
:method => 'GET',
|
|
25
|
+
:path => '/',
|
|
26
|
+
:content_type => "application/x-www-form-urlencoded",
|
|
27
|
+
:content => '',
|
|
28
|
+
:path => '/1/statuses/filter.json',
|
|
29
|
+
:host => 'stream.twitter.com',
|
|
30
|
+
:port => 80,
|
|
31
|
+
:ssl => false,
|
|
32
|
+
:user_agent => 'TwitterStream',
|
|
33
|
+
:timeout => 0,
|
|
34
|
+
:proxy => ENV['HTTP_PROXY'],
|
|
35
|
+
:auth => nil,
|
|
36
|
+
:oauth => {},
|
|
37
|
+
:filters => [],
|
|
38
|
+
:params => {},
|
|
39
|
+
:auto_reconnect => true
|
|
38
40
|
}
|
|
39
41
|
|
|
40
42
|
attr_accessor :code
|
|
@@ -89,6 +91,10 @@ module Twitter
|
|
|
89
91
|
@max_reconnects_callback = block
|
|
90
92
|
end
|
|
91
93
|
|
|
94
|
+
def on_close &block
|
|
95
|
+
@close_callback = block
|
|
96
|
+
end
|
|
97
|
+
|
|
92
98
|
def stop
|
|
93
99
|
@gracefully_closed = true
|
|
94
100
|
close_connection
|
|
@@ -101,20 +107,18 @@ module Twitter
|
|
|
101
107
|
end
|
|
102
108
|
|
|
103
109
|
def unbind
|
|
104
|
-
|
|
105
|
-
|
|
110
|
+
if @state == :stream && !@buffer.empty?
|
|
111
|
+
parse_stream_line(@buffer.flush)
|
|
112
|
+
end
|
|
113
|
+
schedule_reconnect if @options[:auto_reconnect] && !@gracefully_closed
|
|
114
|
+
@close_callback.call if @close_callback
|
|
115
|
+
|
|
106
116
|
end
|
|
107
117
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
end
|
|
113
|
-
rescue Exception => e
|
|
114
|
-
receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
|
|
115
|
-
close_connection
|
|
116
|
-
return
|
|
117
|
-
end
|
|
118
|
+
# Receives raw data from the HTTP connection and pushes it into the
|
|
119
|
+
# HTTP parser which then drives subsequent callbacks.
|
|
120
|
+
def receive_data(data)
|
|
121
|
+
@parser << data
|
|
118
122
|
end
|
|
119
123
|
|
|
120
124
|
def connection_completed
|
|
@@ -175,9 +179,40 @@ module Twitter
|
|
|
175
179
|
def reset_state
|
|
176
180
|
set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
|
|
177
181
|
@code = 0
|
|
178
|
-
@headers =
|
|
182
|
+
@headers = {}
|
|
179
183
|
@state = :init
|
|
180
184
|
@buffer = BufferedTokenizer.new("\r", MAX_LINE_LENGTH)
|
|
185
|
+
@stream = ''
|
|
186
|
+
|
|
187
|
+
@parser = Http::Parser.new
|
|
188
|
+
@parser.on_headers_complete = method(:handle_headers_complete)
|
|
189
|
+
@parser.on_body = method(:receive_stream_data)
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# Called when the status line and all headers have been read from the
|
|
193
|
+
# stream.
|
|
194
|
+
def handle_headers_complete(headers)
|
|
195
|
+
@code = @parser.status_code.to_i
|
|
196
|
+
if @code != 200
|
|
197
|
+
receive_error("invalid status code: #{@code}.")
|
|
198
|
+
end
|
|
199
|
+
self.headers = headers
|
|
200
|
+
@state = :stream
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# Called every time a chunk of data is read from the connection once it has
|
|
204
|
+
# been opened and after the headers have been processed.
|
|
205
|
+
def receive_stream_data(data)
|
|
206
|
+
begin
|
|
207
|
+
@buffer.extract(data).each do |line|
|
|
208
|
+
parse_stream_line(line)
|
|
209
|
+
end
|
|
210
|
+
@stream = ''
|
|
211
|
+
rescue Exception => e
|
|
212
|
+
receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
|
|
213
|
+
close_connection
|
|
214
|
+
return
|
|
215
|
+
end
|
|
181
216
|
end
|
|
182
217
|
|
|
183
218
|
def send_request
|
|
@@ -217,22 +252,18 @@ module Twitter
|
|
|
217
252
|
data << "Content-type: #{@options[:content_type]}"
|
|
218
253
|
data << "Content-length: #{content.length}"
|
|
219
254
|
end
|
|
255
|
+
|
|
256
|
+
if @options[:headers]
|
|
257
|
+
@options[:headers].each do |name,value|
|
|
258
|
+
data << "#{name}: #{value}"
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
220
262
|
data << "\r\n"
|
|
221
263
|
|
|
222
264
|
send_data data.join("\r\n") << content
|
|
223
265
|
end
|
|
224
266
|
|
|
225
|
-
def receive_line ln
|
|
226
|
-
case @state
|
|
227
|
-
when :init
|
|
228
|
-
parse_response_line ln
|
|
229
|
-
when :headers
|
|
230
|
-
parse_header_line ln
|
|
231
|
-
when :stream
|
|
232
|
-
parse_stream_line ln
|
|
233
|
-
end
|
|
234
|
-
end
|
|
235
|
-
|
|
236
267
|
def receive_error e
|
|
237
268
|
@error_callback.call(e) if @error_callback
|
|
238
269
|
end
|
|
@@ -240,34 +271,18 @@ module Twitter
|
|
|
240
271
|
def parse_stream_line ln
|
|
241
272
|
ln.strip!
|
|
242
273
|
unless ln.empty?
|
|
243
|
-
if ln[0,1] == '{'
|
|
244
|
-
@
|
|
274
|
+
if ln[0,1] == '{' || ln[ln.length-1,1] == '}'
|
|
275
|
+
@stream << ln
|
|
276
|
+
if @stream[0,1] == '{' && @stream[@stream.length-1,1] == '}'
|
|
277
|
+
@each_item_callback.call(@stream) if @each_item_callback
|
|
278
|
+
@stream = ''
|
|
279
|
+
end
|
|
245
280
|
end
|
|
246
281
|
end
|
|
247
282
|
end
|
|
248
283
|
|
|
249
|
-
def parse_header_line ln
|
|
250
|
-
ln.strip!
|
|
251
|
-
if ln.empty?
|
|
252
|
-
reset_timeouts if @code == 200
|
|
253
|
-
@state = :stream
|
|
254
|
-
else
|
|
255
|
-
headers << ln
|
|
256
|
-
end
|
|
257
|
-
end
|
|
258
|
-
|
|
259
|
-
def parse_response_line ln
|
|
260
|
-
if ln =~ /\AHTTP\/1\.[01] ([\d]{3})/
|
|
261
|
-
@code = $1.to_i
|
|
262
|
-
@state = :headers
|
|
263
|
-
receive_error("invalid status code: #{@code}. #{ln}") unless @code == 200
|
|
264
|
-
else
|
|
265
|
-
receive_error('invalid response')
|
|
266
|
-
close_connection
|
|
267
|
-
end
|
|
268
|
-
end
|
|
269
|
-
|
|
270
284
|
def reset_timeouts
|
|
285
|
+
set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
|
|
271
286
|
@nf_last_reconnect = @af_last_reconnect = nil
|
|
272
287
|
@reconnect_retries = 0
|
|
273
288
|
end
|
|
@@ -309,15 +324,15 @@ module Twitter
|
|
|
309
324
|
def params
|
|
310
325
|
flat = {}
|
|
311
326
|
@options[:params].merge( :track => @options[:filters] ).each do |param, val|
|
|
312
|
-
next if val.empty?
|
|
327
|
+
next if val.to_s.empty? || (val.respond_to?(:empty?) && val.empty?)
|
|
313
328
|
val = val.join(",") if val.respond_to?(:join)
|
|
314
|
-
flat[
|
|
329
|
+
flat[param.to_s] = val.to_s
|
|
315
330
|
end
|
|
316
331
|
flat
|
|
317
332
|
end
|
|
318
333
|
|
|
319
334
|
def query
|
|
320
|
-
params.map{|
|
|
335
|
+
params.map{|param, value| [escape(param), escape(value)].join("=")}.sort.join("&")
|
|
321
336
|
end
|
|
322
337
|
|
|
323
338
|
def escape str
|
data/spec/spec_helper.rb
CHANGED
|
@@ -26,4 +26,29 @@ def connect_stream(opts={}, &blk)
|
|
|
26
26
|
blk.call if blk
|
|
27
27
|
EM.add_timer(stop_in){ EM.stop }
|
|
28
28
|
}
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def http_response(status_code, status_text, headers, body)
|
|
32
|
+
res = "HTTP/1.1 #{status_code} #{status_text}\r\n"
|
|
33
|
+
headers = {
|
|
34
|
+
"Content-Type"=>"application/json",
|
|
35
|
+
"Transfer-Encoding"=>"chunked"
|
|
36
|
+
}.merge(headers)
|
|
37
|
+
headers.each do |key,value|
|
|
38
|
+
res << "#{key}: #{value}\r\n"
|
|
39
|
+
end
|
|
40
|
+
res << "\r\n"
|
|
41
|
+
if headers["Transfer-Encoding"] == "chunked" && body.kind_of?(Array)
|
|
42
|
+
body.each do |data|
|
|
43
|
+
res << http_chunk(data)
|
|
44
|
+
end
|
|
45
|
+
else
|
|
46
|
+
res << body
|
|
47
|
+
end
|
|
48
|
+
res
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def http_chunk(data)
|
|
52
|
+
# See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
|
|
53
|
+
"#{data.length.to_s(16)}\r\n#{data}\r\n"
|
|
29
54
|
end
|
|
@@ -74,7 +74,9 @@ describe JSONStream do
|
|
|
74
74
|
context "on valid stream" do
|
|
75
75
|
attr_reader :stream
|
|
76
76
|
before :each do
|
|
77
|
-
$
|
|
77
|
+
$body = File.readlines(fixture_path("twitter/tweets.txt"))
|
|
78
|
+
$body.each {|tweet| tweet.strip!; tweet << "\r" }
|
|
79
|
+
$data_to_send = http_response(200,"OK",{},$body)
|
|
78
80
|
$recieved_data = ''
|
|
79
81
|
$close_connection = false
|
|
80
82
|
end
|
|
@@ -92,13 +94,13 @@ describe JSONStream do
|
|
|
92
94
|
it "should parse headers" do
|
|
93
95
|
connect_stream
|
|
94
96
|
stream.code.should == 200
|
|
95
|
-
stream.headers
|
|
97
|
+
stream.headers.keys.map{|k| k.downcase}.should include('content-type')
|
|
96
98
|
end
|
|
97
99
|
|
|
98
100
|
it "should parse headers even after connection close" do
|
|
99
101
|
connect_stream
|
|
100
102
|
stream.code.should == 200
|
|
101
|
-
stream.headers
|
|
103
|
+
stream.headers.keys.map{|k| k.downcase}.should include('content-type')
|
|
102
104
|
end
|
|
103
105
|
|
|
104
106
|
it "should extract records" do
|
|
@@ -106,6 +108,26 @@ describe JSONStream do
|
|
|
106
108
|
$recieved_data.upcase.should include('USER-AGENT: TEST_USER_AGENT')
|
|
107
109
|
end
|
|
108
110
|
|
|
111
|
+
it 'should allow custom headers' do
|
|
112
|
+
connect_stream :headers => { 'From' => 'twitter-stream' }
|
|
113
|
+
$recieved_data.upcase.should include('FROM: TWITTER-STREAM')
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
it "should deliver each item" do
|
|
117
|
+
items = []
|
|
118
|
+
connect_stream do
|
|
119
|
+
stream.each_item do |item|
|
|
120
|
+
items << item
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
# Extract only the tweets from the fixture
|
|
124
|
+
tweets = $body.map{|l| l.strip }.select{|l| l =~ /^\{/ }
|
|
125
|
+
items.size.should == tweets.size
|
|
126
|
+
tweets.each_with_index do |tweet,i|
|
|
127
|
+
items[i].should == tweet
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
109
131
|
it "should send correct user agent" do
|
|
110
132
|
connect_stream
|
|
111
133
|
end
|
|
@@ -118,6 +140,12 @@ describe JSONStream do
|
|
|
118
140
|
end
|
|
119
141
|
end
|
|
120
142
|
|
|
143
|
+
it "should not reconnect on network failure when not configured to auto reconnect" do
|
|
144
|
+
connect_stream(:auto_reconnect => false) do
|
|
145
|
+
stream.should_receive(:reconnect).never
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
121
149
|
it "should reconnect with 0.25 at base" do
|
|
122
150
|
connect_stream do
|
|
123
151
|
stream.should_receive(:reconnect_after).with(0.25)
|
|
@@ -164,6 +192,12 @@ describe JSONStream do
|
|
|
164
192
|
end
|
|
165
193
|
end
|
|
166
194
|
|
|
195
|
+
it "should not reconnect on inactivity when not configured to auto reconnect" do
|
|
196
|
+
connect_stream(:stop_in => 1.5, :auto_reconnect => false) do
|
|
197
|
+
stream.should_receive(:reconnect).never
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
167
201
|
it_should_behave_like "network failure"
|
|
168
202
|
end
|
|
169
203
|
|
|
@@ -186,8 +220,8 @@ describe JSONStream do
|
|
|
186
220
|
context "on application failure" do
|
|
187
221
|
attr_reader :stream
|
|
188
222
|
before :each do
|
|
189
|
-
$data_to_send =
|
|
190
|
-
$close_connection =
|
|
223
|
+
$data_to_send = "HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Basic realm=\"Firehose\"\r\n\r\n"
|
|
224
|
+
$close_connection = false
|
|
191
225
|
end
|
|
192
226
|
|
|
193
227
|
it "should reconnect on application failure 10 at base" do
|
|
@@ -196,6 +230,12 @@ describe JSONStream do
|
|
|
196
230
|
end
|
|
197
231
|
end
|
|
198
232
|
|
|
233
|
+
it "should not reconnect on application failure 10 at base when not configured to auto reconnect" do
|
|
234
|
+
connect_stream(:auto_reconnect => false) do
|
|
235
|
+
stream.should_receive(:reconnect_after).never
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
|
|
199
239
|
it "should reconnect with exponential timeout" do
|
|
200
240
|
connect_stream do
|
|
201
241
|
stream.af_last_reconnect = 160
|
|
@@ -210,4 +250,41 @@ describe JSONStream do
|
|
|
210
250
|
end
|
|
211
251
|
end
|
|
212
252
|
end
|
|
253
|
+
|
|
254
|
+
context "on stream with chunked transfer encoding" do
|
|
255
|
+
attr_reader :stream
|
|
256
|
+
before :each do
|
|
257
|
+
$recieved_data = ''
|
|
258
|
+
$close_connection = false
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
it "should ignore empty lines" do
|
|
262
|
+
body_chunks = ["{\"screen"+"_name\"",":\"user1\"}\r\r\r{","\"id\":9876}\r\r"]
|
|
263
|
+
$data_to_send = http_response(200,"OK",{},body_chunks)
|
|
264
|
+
items = []
|
|
265
|
+
connect_stream do
|
|
266
|
+
stream.each_item do |item|
|
|
267
|
+
items << item
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
items.size.should == 2
|
|
271
|
+
items[0].should == '{"screen_name":"user1"}'
|
|
272
|
+
items[1].should == '{"id":9876}'
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
it "should parse full entities even if split" do
|
|
276
|
+
body_chunks = ["{\"id\"",":1234}\r{","\"id\":9876}"]
|
|
277
|
+
$data_to_send = http_response(200,"OK",{},body_chunks)
|
|
278
|
+
items = []
|
|
279
|
+
connect_stream do
|
|
280
|
+
stream.each_item do |item|
|
|
281
|
+
items << item
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
items.size.should == 2
|
|
285
|
+
items[0].should == '{"id":1234}'
|
|
286
|
+
items[1].should == '{"id":9876}'
|
|
287
|
+
end
|
|
288
|
+
end
|
|
289
|
+
|
|
213
290
|
end
|
data/twitter-stream.gemspec
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Gem::Specification.new do |s|
|
|
4
4
|
s.name = %q{twitter-stream}
|
|
5
|
-
s.version = "0.1.
|
|
5
|
+
s.version = "0.1.14"
|
|
6
6
|
|
|
7
7
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
8
8
|
s.authors = ["Vladimir Kolesnikov"]
|
|
@@ -19,8 +19,9 @@ Gem::Specification.new do |s|
|
|
|
19
19
|
s.rdoc_options = ["--charset=UTF-8"]
|
|
20
20
|
s.extra_rdoc_files = ["README.markdown"]
|
|
21
21
|
|
|
22
|
-
s.add_runtime_dependency('eventmachine', "
|
|
22
|
+
s.add_runtime_dependency('eventmachine', ">= 0.12.8")
|
|
23
23
|
s.add_runtime_dependency('simple_oauth', '~> 0.1.4')
|
|
24
|
+
s.add_runtime_dependency('http_parser.rb', '~> 0.5.1')
|
|
24
25
|
s.add_development_dependency('rspec', "~> 2.5.0")
|
|
25
26
|
|
|
26
27
|
s.files = `git ls-files`.split("\n")
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: twitter-stream
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
hash:
|
|
5
|
-
prerelease:
|
|
4
|
+
hash: 7
|
|
5
|
+
prerelease: false
|
|
6
6
|
segments:
|
|
7
7
|
- 0
|
|
8
8
|
- 1
|
|
9
|
-
-
|
|
10
|
-
version: 0.1.
|
|
9
|
+
- 14
|
|
10
|
+
version: 0.1.14
|
|
11
11
|
platform: ruby
|
|
12
12
|
authors:
|
|
13
13
|
- Vladimir Kolesnikov
|
|
@@ -15,7 +15,8 @@ autorequire:
|
|
|
15
15
|
bindir: bin
|
|
16
16
|
cert_chain: []
|
|
17
17
|
|
|
18
|
-
date: 2010-10-05 00:00:00
|
|
18
|
+
date: 2010-10-05 00:00:00 -07:00
|
|
19
|
+
default_executable:
|
|
19
20
|
dependencies:
|
|
20
21
|
- !ruby/object:Gem::Dependency
|
|
21
22
|
name: eventmachine
|
|
@@ -23,7 +24,7 @@ dependencies:
|
|
|
23
24
|
requirement: &id001 !ruby/object:Gem::Requirement
|
|
24
25
|
none: false
|
|
25
26
|
requirements:
|
|
26
|
-
- -
|
|
27
|
+
- - ">="
|
|
27
28
|
- !ruby/object:Gem::Version
|
|
28
29
|
hash: 63
|
|
29
30
|
segments:
|
|
@@ -50,9 +51,25 @@ dependencies:
|
|
|
50
51
|
type: :runtime
|
|
51
52
|
version_requirements: *id002
|
|
52
53
|
- !ruby/object:Gem::Dependency
|
|
53
|
-
name:
|
|
54
|
+
name: http_parser.rb
|
|
54
55
|
prerelease: false
|
|
55
56
|
requirement: &id003 !ruby/object:Gem::Requirement
|
|
57
|
+
none: false
|
|
58
|
+
requirements:
|
|
59
|
+
- - ~>
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
hash: 9
|
|
62
|
+
segments:
|
|
63
|
+
- 0
|
|
64
|
+
- 5
|
|
65
|
+
- 1
|
|
66
|
+
version: 0.5.1
|
|
67
|
+
type: :runtime
|
|
68
|
+
version_requirements: *id003
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: rspec
|
|
71
|
+
prerelease: false
|
|
72
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
|
56
73
|
none: false
|
|
57
74
|
requirements:
|
|
58
75
|
- - ~>
|
|
@@ -64,7 +81,7 @@ dependencies:
|
|
|
64
81
|
- 0
|
|
65
82
|
version: 2.5.0
|
|
66
83
|
type: :development
|
|
67
|
-
version_requirements: *
|
|
84
|
+
version_requirements: *id004
|
|
68
85
|
description: Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only.
|
|
69
86
|
email: voloko@gmail.com
|
|
70
87
|
executables: []
|
|
@@ -82,11 +99,12 @@ files:
|
|
|
82
99
|
- Rakefile
|
|
83
100
|
- VERSION
|
|
84
101
|
- examples/reader.rb
|
|
85
|
-
- fixtures/twitter/
|
|
102
|
+
- fixtures/twitter/tweets.txt
|
|
86
103
|
- lib/twitter/json_stream.rb
|
|
87
104
|
- spec/spec_helper.rb
|
|
88
105
|
- spec/twitter/json_stream_spec.rb
|
|
89
106
|
- twitter-stream.gemspec
|
|
107
|
+
has_rdoc: true
|
|
90
108
|
homepage: http://github.com/voloko/twitter-stream
|
|
91
109
|
licenses: []
|
|
92
110
|
|
|
@@ -118,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
118
136
|
requirements: []
|
|
119
137
|
|
|
120
138
|
rubyforge_project:
|
|
121
|
-
rubygems_version: 1.7
|
|
139
|
+
rubygems_version: 1.3.7
|
|
122
140
|
signing_key:
|
|
123
141
|
specification_version: 3
|
|
124
142
|
summary: Twitter realtime API client
|