xh5-twitter-stream 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/README.markdown +44 -0
- data/Rakefile +28 -0
- data/VERSION +1 -0
- data/examples/reader.rb +40 -0
- data/fixtures/twitter_stream/basic_http.txt +14 -0
- data/lib/twitter_stream/json_stream.rb +290 -0
- data/spec/spec_helper.rb +27 -0
- data/spec/twitter_stream/json_stream.rb +162 -0
- data/xh5-twitter-stream.gemspec +61 -0
- metadata +115 -0
data/.gitignore
ADDED
data/README.markdown
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
# xh5-twitter-stream
|
2
|
+
|
3
|
+
Forked from original TwitterStream gem. This fork doesn't use Twitter::Module.
|
4
|
+
|
5
|
+
Simple Ruby client library for [twitter streaming API](http://apiwiki.twitter.com/Streaming-API-Documentation).
|
6
|
+
Uses [EventMachine](http://rubyeventmachine.com/) for connection handling. Adheres to twitter's [reconnection guidline](http://apiwiki.twitter.com/Streaming-API-Documentation#Connecting).
|
7
|
+
|
8
|
+
JSON format only.
|
9
|
+
|
10
|
+
## Install
|
11
|
+
|
12
|
+
sudo gem install xh5-twitter-stream -s http://gemcutter.org
|
13
|
+
|
14
|
+
## Usage
|
15
|
+
|
16
|
+
require 'rubygems'
|
17
|
+
require 'twitter_stream/json_stream'
|
18
|
+
|
19
|
+
EventMachine::run {
|
20
|
+
stream = TwitterStream::JSONStream.connect(
|
21
|
+
:path => '/1/statuses/filter.json?track=football',
|
22
|
+
:auth => 'LOGIN:PASSWORD'
|
23
|
+
)
|
24
|
+
|
25
|
+
stream.each_item do |item|
|
26
|
+
# Do someting with unparsed JSON item.
|
27
|
+
end
|
28
|
+
|
29
|
+
stream.on_error do |message|
|
30
|
+
# No need to worry here. It might be an issue with Twitter.
|
31
|
+
# Log message for future reference. JSONStream will try to reconnect after a timeout.
|
32
|
+
end
|
33
|
+
|
34
|
+
stream.on_max_reconnects do |timeout, retries|
|
35
|
+
# Something is wrong on your side. Send yourself an email.
|
36
|
+
end
|
37
|
+
}
|
38
|
+
|
39
|
+
|
40
|
+
## Examples
|
41
|
+
|
42
|
+
Open examples/reader.rb. Replace LOGIN:PASSWORD with your real twitter login and password. And
|
43
|
+
ruby examples/reader.rb
|
44
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
|
3
|
+
gem 'rspec', '= 1.3.0'
|
4
|
+
require 'spec/rake/spectask'
|
5
|
+
|
6
|
+
desc "Run all specs"
|
7
|
+
Spec::Rake::SpecTask.new('spec') do |t|
|
8
|
+
t.spec_files = FileList['spec/**/*.rb']
|
9
|
+
t.spec_opts = %w(-fs --color)
|
10
|
+
end
|
11
|
+
task :default => :spec
|
12
|
+
|
13
|
+
begin
|
14
|
+
require 'jeweler'
|
15
|
+
Jeweler::Tasks.new do |gemspec|
|
16
|
+
gemspec.name = "xh5-twitter-stream"
|
17
|
+
gemspec.summary = "Twitter realtime API client"
|
18
|
+
gemspec.description = "Fork of TwitterStream by Vladimir Koklesnikov. Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only."
|
19
|
+
gemspec.email = "eric@xhfive.com"
|
20
|
+
gemspec.homepage = "http://github.com/erichurst/twitter-stream"
|
21
|
+
gemspec.authors = ["Vladimir Kolesnikov"]
|
22
|
+
gemspec.add_dependency("eventmachine", [">= 0.12.8"])
|
23
|
+
gemspec.add_dependency("roauth", [">= 0.0.2"])
|
24
|
+
gemspec.add_development_dependency("rspec", [">= 1.3.0"])
|
25
|
+
end
|
26
|
+
rescue LoadError
|
27
|
+
puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
|
28
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/examples/reader.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
lib_path = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
|
3
|
+
$LOAD_PATH.unshift lib_path unless $LOAD_PATH.include?(lib_path)
|
4
|
+
|
5
|
+
require 'twitter/json_stream'
|
6
|
+
|
7
|
+
EventMachine::run {
|
8
|
+
stream = Twitter::JSONStream.connect(
|
9
|
+
:path => '/1/statuses/filter.json',
|
10
|
+
:auth => 'LOGIN:PASSWORD',
|
11
|
+
:method => 'POST',
|
12
|
+
:content => 'track=basketball,football,baseball,footy,soccer'
|
13
|
+
)
|
14
|
+
|
15
|
+
stream.each_item do |item|
|
16
|
+
$stdout.print "item: #{item}\n"
|
17
|
+
$stdout.flush
|
18
|
+
end
|
19
|
+
|
20
|
+
stream.on_error do |message|
|
21
|
+
$stdout.print "error: #{message}\n"
|
22
|
+
$stdout.flush
|
23
|
+
end
|
24
|
+
|
25
|
+
stream.on_reconnect do |timeout, retries|
|
26
|
+
$stdout.print "reconnecting in: #{timeout} seconds\n"
|
27
|
+
$stdout.flush
|
28
|
+
end
|
29
|
+
|
30
|
+
stream.on_max_reconnects do |timeout, retries|
|
31
|
+
$stdout.print "Failed after #{retries} failed reconnects\n"
|
32
|
+
$stdout.flush
|
33
|
+
end
|
34
|
+
|
35
|
+
trap('TERM') {
|
36
|
+
stream.stop
|
37
|
+
EventMachine.stop if EventMachine.reactor_running?
|
38
|
+
}
|
39
|
+
}
|
40
|
+
puts "The event loop has ended"
|
@@ -0,0 +1,14 @@
|
|
1
|
+
HTTP/1.1 200 OK
|
2
|
+
Content-Type: application/json
|
3
|
+
Transfer-Encoding: chunked
|
4
|
+
Server: Jetty(6.1.17)
|
5
|
+
|
6
|
+
{"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world! Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}
|
7
|
+
|
8
|
+
{"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch: http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}
|
9
|
+
|
10
|
+
{"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats. http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}
|
11
|
+
|
12
|
+
|
13
|
+
|
14
|
+
{"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}
|
@@ -0,0 +1,290 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'em/buftok'
|
3
|
+
require 'uri'
|
4
|
+
require 'roauth'
|
5
|
+
|
6
|
+
module TwitterStream
|
7
|
+
class JSONStream < EventMachine::Connection
|
8
|
+
MAX_LINE_LENGTH = 1024*1024
|
9
|
+
|
10
|
+
# network failure reconnections
|
11
|
+
NF_RECONNECT_START = 0.25
|
12
|
+
NF_RECONNECT_ADD = 0.25
|
13
|
+
NF_RECONNECT_MAX = 16
|
14
|
+
|
15
|
+
# app failure reconnections
|
16
|
+
AF_RECONNECT_START = 10
|
17
|
+
AF_RECONNECT_MUL = 2
|
18
|
+
|
19
|
+
RECONNECT_MAX = 320
|
20
|
+
RETRIES_MAX = 10
|
21
|
+
|
22
|
+
DEFAULT_OPTIONS = {
|
23
|
+
:method => 'GET',
|
24
|
+
:path => '/',
|
25
|
+
:content_type => "application/x-www-form-urlencoded",
|
26
|
+
:content => '',
|
27
|
+
:path => '/1/statuses/filter.json',
|
28
|
+
:host => 'stream.twitter.com',
|
29
|
+
:port => 80,
|
30
|
+
:ssl => false,
|
31
|
+
:user_agent => 'TwitterStream',
|
32
|
+
:timeout => 0,
|
33
|
+
:proxy => ENV['HTTP_PROXY'],
|
34
|
+
:auth => nil,
|
35
|
+
:oauth => {},
|
36
|
+
:filters => []
|
37
|
+
}
|
38
|
+
|
39
|
+
attr_accessor :code
|
40
|
+
attr_accessor :headers
|
41
|
+
attr_accessor :nf_last_reconnect
|
42
|
+
attr_accessor :af_last_reconnect
|
43
|
+
attr_accessor :reconnect_retries
|
44
|
+
attr_accessor :proxy
|
45
|
+
|
46
|
+
def self.connect options = {}
|
47
|
+
options[:port] = 443 if options[:ssl] && !options.has_key?(:port)
|
48
|
+
options = DEFAULT_OPTIONS.merge(options)
|
49
|
+
|
50
|
+
host = options[:host]
|
51
|
+
port = options[:port]
|
52
|
+
|
53
|
+
if options[:proxy]
|
54
|
+
proxy_uri = URI.parse(options[:proxy])
|
55
|
+
host = proxy_uri.host
|
56
|
+
port = proxy_uri.port
|
57
|
+
end
|
58
|
+
|
59
|
+
connection = EventMachine.connect host, port, self, options
|
60
|
+
connection.start_tls if options[:ssl]
|
61
|
+
connection
|
62
|
+
end
|
63
|
+
|
64
|
+
def initialize options = {}
|
65
|
+
@options = DEFAULT_OPTIONS.merge(options) # merge in case initialize called directly
|
66
|
+
@gracefully_closed = false
|
67
|
+
@nf_last_reconnect = nil
|
68
|
+
@af_last_reconnect = nil
|
69
|
+
@reconnect_retries = 0
|
70
|
+
@immediate_reconnect = false
|
71
|
+
@proxy = URI.parse(options[:proxy]) if options[:proxy]
|
72
|
+
end
|
73
|
+
|
74
|
+
def each_item &block
|
75
|
+
@each_item_callback = block
|
76
|
+
end
|
77
|
+
|
78
|
+
def on_error &block
|
79
|
+
@error_callback = block
|
80
|
+
end
|
81
|
+
|
82
|
+
def on_reconnect &block
|
83
|
+
@reconnect_callback = block
|
84
|
+
end
|
85
|
+
|
86
|
+
def on_max_reconnects &block
|
87
|
+
@max_reconnects_callback = block
|
88
|
+
end
|
89
|
+
|
90
|
+
def stop
|
91
|
+
@gracefully_closed = true
|
92
|
+
close_connection
|
93
|
+
end
|
94
|
+
|
95
|
+
def immediate_reconnect
|
96
|
+
@immediate_reconnect = true
|
97
|
+
@gracefully_closed = false
|
98
|
+
close_connection
|
99
|
+
end
|
100
|
+
|
101
|
+
def unbind
|
102
|
+
receive_line(@buffer.flush) unless @buffer.empty?
|
103
|
+
schedule_reconnect unless @gracefully_closed
|
104
|
+
end
|
105
|
+
|
106
|
+
def receive_data data
|
107
|
+
begin
|
108
|
+
@buffer.extract(data).each do |line|
|
109
|
+
receive_line(line)
|
110
|
+
end
|
111
|
+
rescue Exception => e
|
112
|
+
receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
|
113
|
+
close_connection
|
114
|
+
return
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def connection_completed
|
119
|
+
reset_state
|
120
|
+
send_request
|
121
|
+
end
|
122
|
+
|
123
|
+
protected
|
124
|
+
def schedule_reconnect
|
125
|
+
timeout = reconnect_timeout
|
126
|
+
@reconnect_retries += 1
|
127
|
+
if timeout <= RECONNECT_MAX && @reconnect_retries <= RETRIES_MAX
|
128
|
+
reconnect_after(timeout)
|
129
|
+
else
|
130
|
+
@max_reconnects_callback.call(timeout, @reconnect_retries) if @max_reconnects_callback
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def reconnect_after timeout
|
135
|
+
@reconnect_callback.call(timeout, @reconnect_retries) if @reconnect_callback
|
136
|
+
|
137
|
+
if timeout == 0
|
138
|
+
reconnect @options[:host], @options[:port]
|
139
|
+
else
|
140
|
+
EventMachine.add_timer(timeout) do
|
141
|
+
reconnect @options[:host], @options[:port]
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def reconnect_timeout
|
147
|
+
if @immediate_reconnect
|
148
|
+
@immediate_reconnect = false
|
149
|
+
return 0
|
150
|
+
end
|
151
|
+
|
152
|
+
if (@code == 0) # network failure
|
153
|
+
if @nf_last_reconnect
|
154
|
+
@nf_last_reconnect += NF_RECONNECT_ADD
|
155
|
+
else
|
156
|
+
@nf_last_reconnect = NF_RECONNECT_START
|
157
|
+
end
|
158
|
+
[@nf_last_reconnect,NF_RECONNECT_MAX].min
|
159
|
+
else
|
160
|
+
if @af_last_reconnect
|
161
|
+
@af_last_reconnect *= AF_RECONNECT_MUL
|
162
|
+
else
|
163
|
+
@af_last_reconnect = AF_RECONNECT_START
|
164
|
+
end
|
165
|
+
@af_last_reconnect
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def reset_state
|
170
|
+
set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
|
171
|
+
@code = 0
|
172
|
+
@headers = []
|
173
|
+
@state = :init
|
174
|
+
@buffer = BufferedTokenizer.new("\r", MAX_LINE_LENGTH)
|
175
|
+
end
|
176
|
+
|
177
|
+
def send_request
|
178
|
+
data = []
|
179
|
+
request_uri = @options[:path]
|
180
|
+
|
181
|
+
if @proxy
|
182
|
+
# proxies need the request to be for the full url
|
183
|
+
request_uri = "http#{'s' if @options[:ssl]}://#{@options[:host]}:#{@options[:port]}#{request_uri}"
|
184
|
+
end
|
185
|
+
|
186
|
+
content = @options[:content]
|
187
|
+
|
188
|
+
if !@options[:filters].empty?
|
189
|
+
if @options[:method].to_s.upcase == 'GET'
|
190
|
+
request_uri << "?#{filter_list}"
|
191
|
+
else
|
192
|
+
content = filter_list
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
data << "#{@options[:method]} #{request_uri} HTTP/1.1"
|
197
|
+
data << "Host: #{@options[:host]}"
|
198
|
+
data << "User-Agent: #{@options[:user_agent]}" if @options[:user_agent]
|
199
|
+
|
200
|
+
if @options[:auth]
|
201
|
+
data << "Authorization: Basic #{[@options[:auth]].pack('m').delete("\r\n")}"
|
202
|
+
elsif @options[:oauth]
|
203
|
+
data << "Authorization: #{oauth_header}"
|
204
|
+
end
|
205
|
+
|
206
|
+
if @proxy && @proxy.user
|
207
|
+
data << "Proxy-Authorization: Basic " + ["#{@proxy.user}:#{@proxy.password}"].pack('m').delete("\r\n")
|
208
|
+
end
|
209
|
+
if @options[:method] == 'POST'
|
210
|
+
data << "Content-type: #{@options[:content_type]}"
|
211
|
+
data << "Content-length: #{content.length}"
|
212
|
+
end
|
213
|
+
data << "\r\n"
|
214
|
+
|
215
|
+
send_data data.join("\r\n") << content
|
216
|
+
end
|
217
|
+
|
218
|
+
def receive_line ln
|
219
|
+
case @state
|
220
|
+
when :init
|
221
|
+
parse_response_line ln
|
222
|
+
when :headers
|
223
|
+
parse_header_line ln
|
224
|
+
when :stream
|
225
|
+
parse_stream_line ln
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
def receive_error e
|
230
|
+
@error_callback.call(e) if @error_callback
|
231
|
+
end
|
232
|
+
|
233
|
+
def parse_stream_line ln
|
234
|
+
ln.strip!
|
235
|
+
unless ln.empty?
|
236
|
+
if ln[0,1] == '{'
|
237
|
+
@each_item_callback.call(ln) if @each_item_callback
|
238
|
+
end
|
239
|
+
end
|
240
|
+
end
|
241
|
+
|
242
|
+
def parse_header_line ln
|
243
|
+
ln.strip!
|
244
|
+
if ln.empty?
|
245
|
+
reset_timeouts if @code == 200
|
246
|
+
@state = :stream
|
247
|
+
else
|
248
|
+
headers << ln
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
def parse_response_line ln
|
253
|
+
if ln =~ /\AHTTP\/1\.[01] ([\d]{3})/
|
254
|
+
@code = $1.to_i
|
255
|
+
@state = :headers
|
256
|
+
receive_error("invalid status code: #{@code}. #{ln}") unless @code == 200
|
257
|
+
else
|
258
|
+
receive_error('invalid response')
|
259
|
+
close_connection
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
263
|
+
def reset_timeouts
|
264
|
+
@nf_last_reconnect = @af_last_reconnect = nil
|
265
|
+
@reconnect_retries = 0
|
266
|
+
end
|
267
|
+
|
268
|
+
# :filters => %w(miama lebron jesus)
|
269
|
+
# :oauth => {
|
270
|
+
# :consumer_key => [key],
|
271
|
+
# :consumer_secret => [token],
|
272
|
+
# :access_key => [access key],
|
273
|
+
# :access_secret => [access secret]
|
274
|
+
# }
|
275
|
+
def oauth_header
|
276
|
+
uri = "http://#{@options[:host]}#{@options[:path]}"
|
277
|
+
|
278
|
+
params = {
|
279
|
+
'track' => @options[:filters].join(',')
|
280
|
+
}
|
281
|
+
|
282
|
+
::ROAuth.header(@options[:oauth], uri, params, @options[:method])
|
283
|
+
end
|
284
|
+
|
285
|
+
def filter_list
|
286
|
+
"track=#{@options[:filters].join(',')}"
|
287
|
+
end
|
288
|
+
|
289
|
+
end
|
290
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
$:.unshift File.expand_path("../../lib", __FILE__)
|
3
|
+
|
4
|
+
gem 'rspec', '= 1.3.0'
|
5
|
+
require 'spec'
|
6
|
+
require 'spec/mocks'
|
7
|
+
|
8
|
+
require 'twitter_stream/json_stream'
|
9
|
+
|
10
|
+
def fixture_path(path)
|
11
|
+
File.join(File.dirname(__FILE__), '..', 'fixtures', path)
|
12
|
+
end
|
13
|
+
|
14
|
+
def read_fixture(path)
|
15
|
+
File.read(fixture_path(path))
|
16
|
+
end
|
17
|
+
|
18
|
+
def connect_stream(opts={}, &blk)
|
19
|
+
EM.run {
|
20
|
+
opts.merge!(:host => Host, :port => Port)
|
21
|
+
stop_in = opts.delete(:stop_in) || 0.5
|
22
|
+
EM.start_server Host, Port, JSONServer
|
23
|
+
@stream = JSONStream.connect(opts)
|
24
|
+
blk.call if blk
|
25
|
+
EM.add_timer(stop_in){ EM.stop }
|
26
|
+
}
|
27
|
+
end
|
@@ -0,0 +1,162 @@
|
|
1
|
+
$:.unshift "."
|
2
|
+
require File.dirname(__FILE__) + '/../spec_helper.rb'
|
3
|
+
require 'twitter_stream/json_stream'
|
4
|
+
|
5
|
+
include TwitterStream
|
6
|
+
|
7
|
+
describe JSONStream do
|
8
|
+
|
9
|
+
context "on create" do
|
10
|
+
|
11
|
+
it "should return stream" do
|
12
|
+
EM.should_receive(:connect).and_return('TEST INSTANCE')
|
13
|
+
stream = JSONStream.connect {}
|
14
|
+
stream.should == 'TEST INSTANCE'
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should define default properties" do
|
18
|
+
EM.should_receive(:connect).with do |host, port, handler, opts|
|
19
|
+
host.should == 'stream.twitter.com'
|
20
|
+
port.should == 80
|
21
|
+
opts[:path].should == '/1/statuses/filter.json'
|
22
|
+
opts[:method].should == 'GET'
|
23
|
+
end
|
24
|
+
stream = JSONStream.connect {}
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should connect to the proxy if provided" do
|
28
|
+
EM.should_receive(:connect).with do |host, port, handler, opts|
|
29
|
+
host.should == 'my-proxy'
|
30
|
+
port.should == 8080
|
31
|
+
opts[:host].should == 'stream.twitter.com'
|
32
|
+
opts[:port].should == 80
|
33
|
+
opts[:proxy].should == 'http://my-proxy:8080'
|
34
|
+
end
|
35
|
+
stream = JSONStream.connect(:proxy => "http://my-proxy:8080") {}
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
Host = "127.0.0.1"
|
40
|
+
Port = 9550
|
41
|
+
|
42
|
+
class JSONServer < EM::Connection
|
43
|
+
attr_accessor :data
|
44
|
+
def receive_data data
|
45
|
+
$recieved_data = data
|
46
|
+
send_data $data_to_send
|
47
|
+
EventMachine.next_tick {
|
48
|
+
close_connection if $close_connection
|
49
|
+
}
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
context "on valid stream" do
|
54
|
+
attr_reader :stream
|
55
|
+
before :each do
|
56
|
+
$data_to_send = read_fixture('twitter_stream/basic_http.txt')
|
57
|
+
$recieved_data = ''
|
58
|
+
$close_connection = false
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should parse headers" do
|
62
|
+
connect_stream
|
63
|
+
stream.code.should == 200
|
64
|
+
stream.headers[0].downcase.should include 'content-type'
|
65
|
+
end
|
66
|
+
|
67
|
+
it "should parse headers even after connection close" do
|
68
|
+
connect_stream
|
69
|
+
stream.code.should == 200
|
70
|
+
stream.headers[0].downcase.should include 'content-type'
|
71
|
+
end
|
72
|
+
|
73
|
+
it "should extract records" do
|
74
|
+
connect_stream :user_agent => 'TEST_USER_AGENT'
|
75
|
+
$recieved_data.upcase.should include('USER-AGENT: TEST_USER_AGENT')
|
76
|
+
end
|
77
|
+
|
78
|
+
it "should send correct user agent" do
|
79
|
+
connect_stream
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
context "on network failure" do
|
84
|
+
attr_reader :stream
|
85
|
+
before :each do
|
86
|
+
$data_to_send = ''
|
87
|
+
$close_connection = true
|
88
|
+
end
|
89
|
+
|
90
|
+
it "should timeout on inactivity" do
|
91
|
+
connect_stream :stop_in => 1.5 do
|
92
|
+
stream.should_receive(:reconnect)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
it "should reconnect on network failure" do
|
97
|
+
connect_stream do
|
98
|
+
stream.should_receive(:reconnect)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
it "should reconnect with 0.25 at base" do
|
103
|
+
connect_stream do
|
104
|
+
stream.should_receive(:reconnect_after).with(0.25)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
it "should reconnect with linear timeout" do
|
109
|
+
connect_stream do
|
110
|
+
stream.nf_last_reconnect = 1
|
111
|
+
stream.should_receive(:reconnect_after).with(1.25)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
it "should stop reconnecting after 100 times" do
|
116
|
+
connect_stream do
|
117
|
+
stream.reconnect_retries = 100
|
118
|
+
stream.should_not_receive(:reconnect_after)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
it "should notify after reconnect limit is reached" do
|
123
|
+
timeout, retries = nil, nil
|
124
|
+
connect_stream do
|
125
|
+
stream.on_max_reconnects do |t, r|
|
126
|
+
timeout, retries = t, r
|
127
|
+
end
|
128
|
+
stream.reconnect_retries = 100
|
129
|
+
end
|
130
|
+
timeout.should == 0.25
|
131
|
+
retries.should == 101
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
context "on application failure" do
|
136
|
+
attr_reader :stream
|
137
|
+
before :each do
|
138
|
+
$data_to_send = 'HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Basic realm="Firehose"\r\n\r\n1'
|
139
|
+
$close_connection = true
|
140
|
+
end
|
141
|
+
|
142
|
+
it "should reconnect on application failure 10 at base" do
|
143
|
+
connect_stream do
|
144
|
+
stream.should_receive(:reconnect_after).with(10)
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
it "should reconnect with exponential timeout" do
|
149
|
+
connect_stream do
|
150
|
+
stream.af_last_reconnect = 160
|
151
|
+
stream.should_receive(:reconnect_after).with(320)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
it "should not try to reconnect after limit is reached" do
|
156
|
+
connect_stream do
|
157
|
+
stream.af_last_reconnect = 320
|
158
|
+
stream.should_not_receive(:reconnect_after)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{xh5-twitter-stream}
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Vladimir Kolesnikov"]
|
12
|
+
s.date = %q{2010-10-12}
|
13
|
+
s.description = %q{Fork of TwitterStream by Vladimir Koklesnikov. Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only.}
|
14
|
+
s.email = %q{eric@xhfive.com}
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"README.markdown"
|
17
|
+
]
|
18
|
+
s.files = [
|
19
|
+
".gitignore",
|
20
|
+
"README.markdown",
|
21
|
+
"Rakefile",
|
22
|
+
"VERSION",
|
23
|
+
"examples/reader.rb",
|
24
|
+
"fixtures/twitter_stream/basic_http.txt",
|
25
|
+
"lib/twitter_stream/.json_stream.rb.swp",
|
26
|
+
"lib/twitter_stream/json_stream.rb",
|
27
|
+
"spec/spec_helper.rb",
|
28
|
+
"spec/twitter_stream/json_stream.rb",
|
29
|
+
"xh5-twitter-stream.gemspec"
|
30
|
+
]
|
31
|
+
s.homepage = %q{http://github.com/erichurst/twitter-stream}
|
32
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
33
|
+
s.require_paths = ["lib"]
|
34
|
+
s.rubygems_version = %q{1.3.6}
|
35
|
+
s.summary = %q{Twitter realtime API client}
|
36
|
+
s.test_files = [
|
37
|
+
"spec/spec_helper.rb",
|
38
|
+
"spec/twitter_stream/json_stream.rb",
|
39
|
+
"examples/reader.rb"
|
40
|
+
]
|
41
|
+
|
42
|
+
if s.respond_to? :specification_version then
|
43
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
44
|
+
s.specification_version = 3
|
45
|
+
|
46
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
47
|
+
s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.8"])
|
48
|
+
s.add_runtime_dependency(%q<roauth>, [">= 0.0.2"])
|
49
|
+
s.add_development_dependency(%q<rspec>, [">= 1.3.0"])
|
50
|
+
else
|
51
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
|
52
|
+
s.add_dependency(%q<roauth>, [">= 0.0.2"])
|
53
|
+
s.add_dependency(%q<rspec>, [">= 1.3.0"])
|
54
|
+
end
|
55
|
+
else
|
56
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
|
57
|
+
s.add_dependency(%q<roauth>, [">= 0.0.2"])
|
58
|
+
s.add_dependency(%q<rspec>, [">= 1.3.0"])
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
metadata
ADDED
@@ -0,0 +1,115 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: xh5-twitter-stream
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
version: 0.1.0
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Vladimir Kolesnikov
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2010-10-12 00:00:00 -05:00
|
18
|
+
default_executable:
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: eventmachine
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
segments:
|
28
|
+
- 0
|
29
|
+
- 12
|
30
|
+
- 8
|
31
|
+
version: 0.12.8
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: roauth
|
36
|
+
prerelease: false
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
segments:
|
42
|
+
- 0
|
43
|
+
- 0
|
44
|
+
- 2
|
45
|
+
version: 0.0.2
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: rspec
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
segments:
|
56
|
+
- 1
|
57
|
+
- 3
|
58
|
+
- 0
|
59
|
+
version: 1.3.0
|
60
|
+
type: :development
|
61
|
+
version_requirements: *id003
|
62
|
+
description: Fork of TwitterStream by Vladimir Koklesnikov. Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only.
|
63
|
+
email: eric@xhfive.com
|
64
|
+
executables: []
|
65
|
+
|
66
|
+
extensions: []
|
67
|
+
|
68
|
+
extra_rdoc_files:
|
69
|
+
- README.markdown
|
70
|
+
files:
|
71
|
+
- .gitignore
|
72
|
+
- README.markdown
|
73
|
+
- Rakefile
|
74
|
+
- VERSION
|
75
|
+
- examples/reader.rb
|
76
|
+
- fixtures/twitter_stream/basic_http.txt
|
77
|
+
- lib/twitter_stream/.json_stream.rb.swp
|
78
|
+
- lib/twitter_stream/json_stream.rb
|
79
|
+
- spec/spec_helper.rb
|
80
|
+
- spec/twitter_stream/json_stream.rb
|
81
|
+
- xh5-twitter-stream.gemspec
|
82
|
+
has_rdoc: true
|
83
|
+
homepage: http://github.com/erichurst/twitter-stream
|
84
|
+
licenses: []
|
85
|
+
|
86
|
+
post_install_message:
|
87
|
+
rdoc_options:
|
88
|
+
- --charset=UTF-8
|
89
|
+
require_paths:
|
90
|
+
- lib
|
91
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
segments:
|
96
|
+
- 0
|
97
|
+
version: "0"
|
98
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
segments:
|
103
|
+
- 0
|
104
|
+
version: "0"
|
105
|
+
requirements: []
|
106
|
+
|
107
|
+
rubyforge_project:
|
108
|
+
rubygems_version: 1.3.6
|
109
|
+
signing_key:
|
110
|
+
specification_version: 3
|
111
|
+
summary: Twitter realtime API client
|
112
|
+
test_files:
|
113
|
+
- spec/spec_helper.rb
|
114
|
+
- spec/twitter_stream/json_stream.rb
|
115
|
+
- examples/reader.rb
|