xh5-twitter-stream 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/README.markdown +44 -0
- data/Rakefile +28 -0
- data/VERSION +1 -0
- data/examples/reader.rb +40 -0
- data/fixtures/twitter_stream/basic_http.txt +14 -0
- data/lib/twitter_stream/json_stream.rb +290 -0
- data/spec/spec_helper.rb +27 -0
- data/spec/twitter_stream/json_stream.rb +162 -0
- data/xh5-twitter-stream.gemspec +61 -0
- metadata +115 -0
data/.gitignore
ADDED
data/README.markdown
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# xh5-twitter-stream
|
|
2
|
+
|
|
3
|
+
Forked from original TwitterStream gem. This fork doesn't use Twitter::Module.
|
|
4
|
+
|
|
5
|
+
Simple Ruby client library for [twitter streaming API](http://apiwiki.twitter.com/Streaming-API-Documentation).
|
|
6
|
+
Uses [EventMachine](http://rubyeventmachine.com/) for connection handling. Adheres to twitter's [reconnection guidline](http://apiwiki.twitter.com/Streaming-API-Documentation#Connecting).
|
|
7
|
+
|
|
8
|
+
JSON format only.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
sudo gem install xh5-twitter-stream -s http://gemcutter.org
|
|
13
|
+
|
|
14
|
+
## Usage
|
|
15
|
+
|
|
16
|
+
require 'rubygems'
|
|
17
|
+
require 'twitter_stream/json_stream'
|
|
18
|
+
|
|
19
|
+
EventMachine::run {
|
|
20
|
+
stream = TwitterStream::JSONStream.connect(
|
|
21
|
+
:path => '/1/statuses/filter.json?track=football',
|
|
22
|
+
:auth => 'LOGIN:PASSWORD'
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
stream.each_item do |item|
|
|
26
|
+
# Do someting with unparsed JSON item.
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
stream.on_error do |message|
|
|
30
|
+
# No need to worry here. It might be an issue with Twitter.
|
|
31
|
+
# Log message for future reference. JSONStream will try to reconnect after a timeout.
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
stream.on_max_reconnects do |timeout, retries|
|
|
35
|
+
# Something is wrong on your side. Send yourself an email.
|
|
36
|
+
end
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
## Examples
|
|
41
|
+
|
|
42
|
+
Open examples/reader.rb. Replace LOGIN:PASSWORD with your real twitter login and password. And
|
|
43
|
+
ruby examples/reader.rb
|
|
44
|
+
|
data/Rakefile
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
|
|
3
|
+
gem 'rspec', '= 1.3.0'
|
|
4
|
+
require 'spec/rake/spectask'
|
|
5
|
+
|
|
6
|
+
desc "Run all specs"
|
|
7
|
+
Spec::Rake::SpecTask.new('spec') do |t|
|
|
8
|
+
t.spec_files = FileList['spec/**/*.rb']
|
|
9
|
+
t.spec_opts = %w(-fs --color)
|
|
10
|
+
end
|
|
11
|
+
task :default => :spec
|
|
12
|
+
|
|
13
|
+
begin
|
|
14
|
+
require 'jeweler'
|
|
15
|
+
Jeweler::Tasks.new do |gemspec|
|
|
16
|
+
gemspec.name = "xh5-twitter-stream"
|
|
17
|
+
gemspec.summary = "Twitter realtime API client"
|
|
18
|
+
gemspec.description = "Fork of TwitterStream by Vladimir Koklesnikov. Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only."
|
|
19
|
+
gemspec.email = "eric@xhfive.com"
|
|
20
|
+
gemspec.homepage = "http://github.com/erichurst/twitter-stream"
|
|
21
|
+
gemspec.authors = ["Vladimir Kolesnikov"]
|
|
22
|
+
gemspec.add_dependency("eventmachine", [">= 0.12.8"])
|
|
23
|
+
gemspec.add_dependency("roauth", [">= 0.0.2"])
|
|
24
|
+
gemspec.add_development_dependency("rspec", [">= 1.3.0"])
|
|
25
|
+
end
|
|
26
|
+
rescue LoadError
|
|
27
|
+
puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
|
|
28
|
+
end
|
data/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.1.0
|
data/examples/reader.rb
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
lib_path = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
|
|
3
|
+
$LOAD_PATH.unshift lib_path unless $LOAD_PATH.include?(lib_path)
|
|
4
|
+
|
|
5
|
+
require 'twitter/json_stream'
|
|
6
|
+
|
|
7
|
+
EventMachine::run {
|
|
8
|
+
stream = Twitter::JSONStream.connect(
|
|
9
|
+
:path => '/1/statuses/filter.json',
|
|
10
|
+
:auth => 'LOGIN:PASSWORD',
|
|
11
|
+
:method => 'POST',
|
|
12
|
+
:content => 'track=basketball,football,baseball,footy,soccer'
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
stream.each_item do |item|
|
|
16
|
+
$stdout.print "item: #{item}\n"
|
|
17
|
+
$stdout.flush
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
stream.on_error do |message|
|
|
21
|
+
$stdout.print "error: #{message}\n"
|
|
22
|
+
$stdout.flush
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
stream.on_reconnect do |timeout, retries|
|
|
26
|
+
$stdout.print "reconnecting in: #{timeout} seconds\n"
|
|
27
|
+
$stdout.flush
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
stream.on_max_reconnects do |timeout, retries|
|
|
31
|
+
$stdout.print "Failed after #{retries} failed reconnects\n"
|
|
32
|
+
$stdout.flush
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
trap('TERM') {
|
|
36
|
+
stream.stop
|
|
37
|
+
EventMachine.stop if EventMachine.reactor_running?
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
puts "The event loop has ended"
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
HTTP/1.1 200 OK
|
|
2
|
+
Content-Type: application/json
|
|
3
|
+
Transfer-Encoding: chunked
|
|
4
|
+
Server: Jetty(6.1.17)
|
|
5
|
+
|
|
6
|
+
{"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world! Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}
|
|
7
|
+
|
|
8
|
+
{"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch: http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}
|
|
9
|
+
|
|
10
|
+
{"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats. http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
{"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
require 'eventmachine'
|
|
2
|
+
require 'em/buftok'
|
|
3
|
+
require 'uri'
|
|
4
|
+
require 'roauth'
|
|
5
|
+
|
|
6
|
+
module TwitterStream
|
|
7
|
+
class JSONStream < EventMachine::Connection
|
|
8
|
+
MAX_LINE_LENGTH = 1024*1024
|
|
9
|
+
|
|
10
|
+
# network failure reconnections
|
|
11
|
+
NF_RECONNECT_START = 0.25
|
|
12
|
+
NF_RECONNECT_ADD = 0.25
|
|
13
|
+
NF_RECONNECT_MAX = 16
|
|
14
|
+
|
|
15
|
+
# app failure reconnections
|
|
16
|
+
AF_RECONNECT_START = 10
|
|
17
|
+
AF_RECONNECT_MUL = 2
|
|
18
|
+
|
|
19
|
+
RECONNECT_MAX = 320
|
|
20
|
+
RETRIES_MAX = 10
|
|
21
|
+
|
|
22
|
+
DEFAULT_OPTIONS = {
|
|
23
|
+
:method => 'GET',
|
|
24
|
+
:path => '/',
|
|
25
|
+
:content_type => "application/x-www-form-urlencoded",
|
|
26
|
+
:content => '',
|
|
27
|
+
:path => '/1/statuses/filter.json',
|
|
28
|
+
:host => 'stream.twitter.com',
|
|
29
|
+
:port => 80,
|
|
30
|
+
:ssl => false,
|
|
31
|
+
:user_agent => 'TwitterStream',
|
|
32
|
+
:timeout => 0,
|
|
33
|
+
:proxy => ENV['HTTP_PROXY'],
|
|
34
|
+
:auth => nil,
|
|
35
|
+
:oauth => {},
|
|
36
|
+
:filters => []
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
attr_accessor :code
|
|
40
|
+
attr_accessor :headers
|
|
41
|
+
attr_accessor :nf_last_reconnect
|
|
42
|
+
attr_accessor :af_last_reconnect
|
|
43
|
+
attr_accessor :reconnect_retries
|
|
44
|
+
attr_accessor :proxy
|
|
45
|
+
|
|
46
|
+
def self.connect options = {}
|
|
47
|
+
options[:port] = 443 if options[:ssl] && !options.has_key?(:port)
|
|
48
|
+
options = DEFAULT_OPTIONS.merge(options)
|
|
49
|
+
|
|
50
|
+
host = options[:host]
|
|
51
|
+
port = options[:port]
|
|
52
|
+
|
|
53
|
+
if options[:proxy]
|
|
54
|
+
proxy_uri = URI.parse(options[:proxy])
|
|
55
|
+
host = proxy_uri.host
|
|
56
|
+
port = proxy_uri.port
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
connection = EventMachine.connect host, port, self, options
|
|
60
|
+
connection.start_tls if options[:ssl]
|
|
61
|
+
connection
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def initialize options = {}
|
|
65
|
+
@options = DEFAULT_OPTIONS.merge(options) # merge in case initialize called directly
|
|
66
|
+
@gracefully_closed = false
|
|
67
|
+
@nf_last_reconnect = nil
|
|
68
|
+
@af_last_reconnect = nil
|
|
69
|
+
@reconnect_retries = 0
|
|
70
|
+
@immediate_reconnect = false
|
|
71
|
+
@proxy = URI.parse(options[:proxy]) if options[:proxy]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def each_item &block
|
|
75
|
+
@each_item_callback = block
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def on_error &block
|
|
79
|
+
@error_callback = block
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def on_reconnect &block
|
|
83
|
+
@reconnect_callback = block
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def on_max_reconnects &block
|
|
87
|
+
@max_reconnects_callback = block
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def stop
|
|
91
|
+
@gracefully_closed = true
|
|
92
|
+
close_connection
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def immediate_reconnect
|
|
96
|
+
@immediate_reconnect = true
|
|
97
|
+
@gracefully_closed = false
|
|
98
|
+
close_connection
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def unbind
|
|
102
|
+
receive_line(@buffer.flush) unless @buffer.empty?
|
|
103
|
+
schedule_reconnect unless @gracefully_closed
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def receive_data data
|
|
107
|
+
begin
|
|
108
|
+
@buffer.extract(data).each do |line|
|
|
109
|
+
receive_line(line)
|
|
110
|
+
end
|
|
111
|
+
rescue Exception => e
|
|
112
|
+
receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
|
|
113
|
+
close_connection
|
|
114
|
+
return
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def connection_completed
|
|
119
|
+
reset_state
|
|
120
|
+
send_request
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
protected
|
|
124
|
+
def schedule_reconnect
|
|
125
|
+
timeout = reconnect_timeout
|
|
126
|
+
@reconnect_retries += 1
|
|
127
|
+
if timeout <= RECONNECT_MAX && @reconnect_retries <= RETRIES_MAX
|
|
128
|
+
reconnect_after(timeout)
|
|
129
|
+
else
|
|
130
|
+
@max_reconnects_callback.call(timeout, @reconnect_retries) if @max_reconnects_callback
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def reconnect_after timeout
|
|
135
|
+
@reconnect_callback.call(timeout, @reconnect_retries) if @reconnect_callback
|
|
136
|
+
|
|
137
|
+
if timeout == 0
|
|
138
|
+
reconnect @options[:host], @options[:port]
|
|
139
|
+
else
|
|
140
|
+
EventMachine.add_timer(timeout) do
|
|
141
|
+
reconnect @options[:host], @options[:port]
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def reconnect_timeout
|
|
147
|
+
if @immediate_reconnect
|
|
148
|
+
@immediate_reconnect = false
|
|
149
|
+
return 0
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
if (@code == 0) # network failure
|
|
153
|
+
if @nf_last_reconnect
|
|
154
|
+
@nf_last_reconnect += NF_RECONNECT_ADD
|
|
155
|
+
else
|
|
156
|
+
@nf_last_reconnect = NF_RECONNECT_START
|
|
157
|
+
end
|
|
158
|
+
[@nf_last_reconnect,NF_RECONNECT_MAX].min
|
|
159
|
+
else
|
|
160
|
+
if @af_last_reconnect
|
|
161
|
+
@af_last_reconnect *= AF_RECONNECT_MUL
|
|
162
|
+
else
|
|
163
|
+
@af_last_reconnect = AF_RECONNECT_START
|
|
164
|
+
end
|
|
165
|
+
@af_last_reconnect
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def reset_state
|
|
170
|
+
set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
|
|
171
|
+
@code = 0
|
|
172
|
+
@headers = []
|
|
173
|
+
@state = :init
|
|
174
|
+
@buffer = BufferedTokenizer.new("\r", MAX_LINE_LENGTH)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def send_request
|
|
178
|
+
data = []
|
|
179
|
+
request_uri = @options[:path]
|
|
180
|
+
|
|
181
|
+
if @proxy
|
|
182
|
+
# proxies need the request to be for the full url
|
|
183
|
+
request_uri = "http#{'s' if @options[:ssl]}://#{@options[:host]}:#{@options[:port]}#{request_uri}"
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
content = @options[:content]
|
|
187
|
+
|
|
188
|
+
if !@options[:filters].empty?
|
|
189
|
+
if @options[:method].to_s.upcase == 'GET'
|
|
190
|
+
request_uri << "?#{filter_list}"
|
|
191
|
+
else
|
|
192
|
+
content = filter_list
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
data << "#{@options[:method]} #{request_uri} HTTP/1.1"
|
|
197
|
+
data << "Host: #{@options[:host]}"
|
|
198
|
+
data << "User-Agent: #{@options[:user_agent]}" if @options[:user_agent]
|
|
199
|
+
|
|
200
|
+
if @options[:auth]
|
|
201
|
+
data << "Authorization: Basic #{[@options[:auth]].pack('m').delete("\r\n")}"
|
|
202
|
+
elsif @options[:oauth]
|
|
203
|
+
data << "Authorization: #{oauth_header}"
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
if @proxy && @proxy.user
|
|
207
|
+
data << "Proxy-Authorization: Basic " + ["#{@proxy.user}:#{@proxy.password}"].pack('m').delete("\r\n")
|
|
208
|
+
end
|
|
209
|
+
if @options[:method] == 'POST'
|
|
210
|
+
data << "Content-type: #{@options[:content_type]}"
|
|
211
|
+
data << "Content-length: #{content.length}"
|
|
212
|
+
end
|
|
213
|
+
data << "\r\n"
|
|
214
|
+
|
|
215
|
+
send_data data.join("\r\n") << content
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def receive_line ln
|
|
219
|
+
case @state
|
|
220
|
+
when :init
|
|
221
|
+
parse_response_line ln
|
|
222
|
+
when :headers
|
|
223
|
+
parse_header_line ln
|
|
224
|
+
when :stream
|
|
225
|
+
parse_stream_line ln
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def receive_error e
|
|
230
|
+
@error_callback.call(e) if @error_callback
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def parse_stream_line ln
|
|
234
|
+
ln.strip!
|
|
235
|
+
unless ln.empty?
|
|
236
|
+
if ln[0,1] == '{'
|
|
237
|
+
@each_item_callback.call(ln) if @each_item_callback
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def parse_header_line ln
|
|
243
|
+
ln.strip!
|
|
244
|
+
if ln.empty?
|
|
245
|
+
reset_timeouts if @code == 200
|
|
246
|
+
@state = :stream
|
|
247
|
+
else
|
|
248
|
+
headers << ln
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def parse_response_line ln
|
|
253
|
+
if ln =~ /\AHTTP\/1\.[01] ([\d]{3})/
|
|
254
|
+
@code = $1.to_i
|
|
255
|
+
@state = :headers
|
|
256
|
+
receive_error("invalid status code: #{@code}. #{ln}") unless @code == 200
|
|
257
|
+
else
|
|
258
|
+
receive_error('invalid response')
|
|
259
|
+
close_connection
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def reset_timeouts
|
|
264
|
+
@nf_last_reconnect = @af_last_reconnect = nil
|
|
265
|
+
@reconnect_retries = 0
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# :filters => %w(miama lebron jesus)
|
|
269
|
+
# :oauth => {
|
|
270
|
+
# :consumer_key => [key],
|
|
271
|
+
# :consumer_secret => [token],
|
|
272
|
+
# :access_key => [access key],
|
|
273
|
+
# :access_secret => [access secret]
|
|
274
|
+
# }
|
|
275
|
+
def oauth_header
|
|
276
|
+
uri = "http://#{@options[:host]}#{@options[:path]}"
|
|
277
|
+
|
|
278
|
+
params = {
|
|
279
|
+
'track' => @options[:filters].join(',')
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
::ROAuth.header(@options[:oauth], uri, params, @options[:method])
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
def filter_list
|
|
286
|
+
"track=#{@options[:filters].join(',')}"
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
end
|
|
290
|
+
end
|
data/spec/spec_helper.rb
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
$:.unshift File.expand_path("../../lib", __FILE__)
|
|
3
|
+
|
|
4
|
+
gem 'rspec', '= 1.3.0'
|
|
5
|
+
require 'spec'
|
|
6
|
+
require 'spec/mocks'
|
|
7
|
+
|
|
8
|
+
require 'twitter_stream/json_stream'
|
|
9
|
+
|
|
10
|
+
def fixture_path(path)
|
|
11
|
+
File.join(File.dirname(__FILE__), '..', 'fixtures', path)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def read_fixture(path)
|
|
15
|
+
File.read(fixture_path(path))
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def connect_stream(opts={}, &blk)
|
|
19
|
+
EM.run {
|
|
20
|
+
opts.merge!(:host => Host, :port => Port)
|
|
21
|
+
stop_in = opts.delete(:stop_in) || 0.5
|
|
22
|
+
EM.start_server Host, Port, JSONServer
|
|
23
|
+
@stream = JSONStream.connect(opts)
|
|
24
|
+
blk.call if blk
|
|
25
|
+
EM.add_timer(stop_in){ EM.stop }
|
|
26
|
+
}
|
|
27
|
+
end
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
$:.unshift "."
|
|
2
|
+
require File.dirname(__FILE__) + '/../spec_helper.rb'
|
|
3
|
+
require 'twitter_stream/json_stream'
|
|
4
|
+
|
|
5
|
+
include TwitterStream
|
|
6
|
+
|
|
7
|
+
describe JSONStream do
|
|
8
|
+
|
|
9
|
+
context "on create" do
|
|
10
|
+
|
|
11
|
+
it "should return stream" do
|
|
12
|
+
EM.should_receive(:connect).and_return('TEST INSTANCE')
|
|
13
|
+
stream = JSONStream.connect {}
|
|
14
|
+
stream.should == 'TEST INSTANCE'
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
it "should define default properties" do
|
|
18
|
+
EM.should_receive(:connect).with do |host, port, handler, opts|
|
|
19
|
+
host.should == 'stream.twitter.com'
|
|
20
|
+
port.should == 80
|
|
21
|
+
opts[:path].should == '/1/statuses/filter.json'
|
|
22
|
+
opts[:method].should == 'GET'
|
|
23
|
+
end
|
|
24
|
+
stream = JSONStream.connect {}
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
it "should connect to the proxy if provided" do
|
|
28
|
+
EM.should_receive(:connect).with do |host, port, handler, opts|
|
|
29
|
+
host.should == 'my-proxy'
|
|
30
|
+
port.should == 8080
|
|
31
|
+
opts[:host].should == 'stream.twitter.com'
|
|
32
|
+
opts[:port].should == 80
|
|
33
|
+
opts[:proxy].should == 'http://my-proxy:8080'
|
|
34
|
+
end
|
|
35
|
+
stream = JSONStream.connect(:proxy => "http://my-proxy:8080") {}
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
Host = "127.0.0.1"
|
|
40
|
+
Port = 9550
|
|
41
|
+
|
|
42
|
+
class JSONServer < EM::Connection
|
|
43
|
+
attr_accessor :data
|
|
44
|
+
def receive_data data
|
|
45
|
+
$recieved_data = data
|
|
46
|
+
send_data $data_to_send
|
|
47
|
+
EventMachine.next_tick {
|
|
48
|
+
close_connection if $close_connection
|
|
49
|
+
}
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
context "on valid stream" do
|
|
54
|
+
attr_reader :stream
|
|
55
|
+
before :each do
|
|
56
|
+
$data_to_send = read_fixture('twitter_stream/basic_http.txt')
|
|
57
|
+
$recieved_data = ''
|
|
58
|
+
$close_connection = false
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
it "should parse headers" do
|
|
62
|
+
connect_stream
|
|
63
|
+
stream.code.should == 200
|
|
64
|
+
stream.headers[0].downcase.should include 'content-type'
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
it "should parse headers even after connection close" do
|
|
68
|
+
connect_stream
|
|
69
|
+
stream.code.should == 200
|
|
70
|
+
stream.headers[0].downcase.should include 'content-type'
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it "should extract records" do
|
|
74
|
+
connect_stream :user_agent => 'TEST_USER_AGENT'
|
|
75
|
+
$recieved_data.upcase.should include('USER-AGENT: TEST_USER_AGENT')
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
it "should send correct user agent" do
|
|
79
|
+
connect_stream
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
context "on network failure" do
|
|
84
|
+
attr_reader :stream
|
|
85
|
+
before :each do
|
|
86
|
+
$data_to_send = ''
|
|
87
|
+
$close_connection = true
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
it "should timeout on inactivity" do
|
|
91
|
+
connect_stream :stop_in => 1.5 do
|
|
92
|
+
stream.should_receive(:reconnect)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
it "should reconnect on network failure" do
|
|
97
|
+
connect_stream do
|
|
98
|
+
stream.should_receive(:reconnect)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
it "should reconnect with 0.25 at base" do
|
|
103
|
+
connect_stream do
|
|
104
|
+
stream.should_receive(:reconnect_after).with(0.25)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
it "should reconnect with linear timeout" do
|
|
109
|
+
connect_stream do
|
|
110
|
+
stream.nf_last_reconnect = 1
|
|
111
|
+
stream.should_receive(:reconnect_after).with(1.25)
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
it "should stop reconnecting after 100 times" do
|
|
116
|
+
connect_stream do
|
|
117
|
+
stream.reconnect_retries = 100
|
|
118
|
+
stream.should_not_receive(:reconnect_after)
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
it "should notify after reconnect limit is reached" do
|
|
123
|
+
timeout, retries = nil, nil
|
|
124
|
+
connect_stream do
|
|
125
|
+
stream.on_max_reconnects do |t, r|
|
|
126
|
+
timeout, retries = t, r
|
|
127
|
+
end
|
|
128
|
+
stream.reconnect_retries = 100
|
|
129
|
+
end
|
|
130
|
+
timeout.should == 0.25
|
|
131
|
+
retries.should == 101
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
context "on application failure" do
|
|
136
|
+
attr_reader :stream
|
|
137
|
+
before :each do
|
|
138
|
+
$data_to_send = 'HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Basic realm="Firehose"\r\n\r\n1'
|
|
139
|
+
$close_connection = true
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
it "should reconnect on application failure 10 at base" do
|
|
143
|
+
connect_stream do
|
|
144
|
+
stream.should_receive(:reconnect_after).with(10)
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
it "should reconnect with exponential timeout" do
|
|
149
|
+
connect_stream do
|
|
150
|
+
stream.af_last_reconnect = 160
|
|
151
|
+
stream.should_receive(:reconnect_after).with(320)
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
it "should not try to reconnect after limit is reached" do
|
|
156
|
+
connect_stream do
|
|
157
|
+
stream.af_last_reconnect = 320
|
|
158
|
+
stream.should_not_receive(:reconnect_after)
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Generated by jeweler
|
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
|
4
|
+
# -*- encoding: utf-8 -*-
|
|
5
|
+
|
|
6
|
+
Gem::Specification.new do |s|
|
|
7
|
+
s.name = %q{xh5-twitter-stream}
|
|
8
|
+
s.version = "0.1.0"
|
|
9
|
+
|
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
11
|
+
s.authors = ["Vladimir Kolesnikov"]
|
|
12
|
+
s.date = %q{2010-10-12}
|
|
13
|
+
s.description = %q{Fork of TwitterStream by Vladimir Koklesnikov. Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only.}
|
|
14
|
+
s.email = %q{eric@xhfive.com}
|
|
15
|
+
s.extra_rdoc_files = [
|
|
16
|
+
"README.markdown"
|
|
17
|
+
]
|
|
18
|
+
s.files = [
|
|
19
|
+
".gitignore",
|
|
20
|
+
"README.markdown",
|
|
21
|
+
"Rakefile",
|
|
22
|
+
"VERSION",
|
|
23
|
+
"examples/reader.rb",
|
|
24
|
+
"fixtures/twitter_stream/basic_http.txt",
|
|
25
|
+
"lib/twitter_stream/.json_stream.rb.swp",
|
|
26
|
+
"lib/twitter_stream/json_stream.rb",
|
|
27
|
+
"spec/spec_helper.rb",
|
|
28
|
+
"spec/twitter_stream/json_stream.rb",
|
|
29
|
+
"xh5-twitter-stream.gemspec"
|
|
30
|
+
]
|
|
31
|
+
s.homepage = %q{http://github.com/erichurst/twitter-stream}
|
|
32
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
|
33
|
+
s.require_paths = ["lib"]
|
|
34
|
+
s.rubygems_version = %q{1.3.6}
|
|
35
|
+
s.summary = %q{Twitter realtime API client}
|
|
36
|
+
s.test_files = [
|
|
37
|
+
"spec/spec_helper.rb",
|
|
38
|
+
"spec/twitter_stream/json_stream.rb",
|
|
39
|
+
"examples/reader.rb"
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
if s.respond_to? :specification_version then
|
|
43
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
|
44
|
+
s.specification_version = 3
|
|
45
|
+
|
|
46
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
|
47
|
+
s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.8"])
|
|
48
|
+
s.add_runtime_dependency(%q<roauth>, [">= 0.0.2"])
|
|
49
|
+
s.add_development_dependency(%q<rspec>, [">= 1.3.0"])
|
|
50
|
+
else
|
|
51
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
|
|
52
|
+
s.add_dependency(%q<roauth>, [">= 0.0.2"])
|
|
53
|
+
s.add_dependency(%q<rspec>, [">= 1.3.0"])
|
|
54
|
+
end
|
|
55
|
+
else
|
|
56
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
|
|
57
|
+
s.add_dependency(%q<roauth>, [">= 0.0.2"])
|
|
58
|
+
s.add_dependency(%q<rspec>, [">= 1.3.0"])
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
metadata
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: xh5-twitter-stream
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
prerelease: false
|
|
5
|
+
segments:
|
|
6
|
+
- 0
|
|
7
|
+
- 1
|
|
8
|
+
- 0
|
|
9
|
+
version: 0.1.0
|
|
10
|
+
platform: ruby
|
|
11
|
+
authors:
|
|
12
|
+
- Vladimir Kolesnikov
|
|
13
|
+
autorequire:
|
|
14
|
+
bindir: bin
|
|
15
|
+
cert_chain: []
|
|
16
|
+
|
|
17
|
+
date: 2010-10-12 00:00:00 -05:00
|
|
18
|
+
default_executable:
|
|
19
|
+
dependencies:
|
|
20
|
+
- !ruby/object:Gem::Dependency
|
|
21
|
+
name: eventmachine
|
|
22
|
+
prerelease: false
|
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
|
24
|
+
requirements:
|
|
25
|
+
- - ">="
|
|
26
|
+
- !ruby/object:Gem::Version
|
|
27
|
+
segments:
|
|
28
|
+
- 0
|
|
29
|
+
- 12
|
|
30
|
+
- 8
|
|
31
|
+
version: 0.12.8
|
|
32
|
+
type: :runtime
|
|
33
|
+
version_requirements: *id001
|
|
34
|
+
- !ruby/object:Gem::Dependency
|
|
35
|
+
name: roauth
|
|
36
|
+
prerelease: false
|
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
|
38
|
+
requirements:
|
|
39
|
+
- - ">="
|
|
40
|
+
- !ruby/object:Gem::Version
|
|
41
|
+
segments:
|
|
42
|
+
- 0
|
|
43
|
+
- 0
|
|
44
|
+
- 2
|
|
45
|
+
version: 0.0.2
|
|
46
|
+
type: :runtime
|
|
47
|
+
version_requirements: *id002
|
|
48
|
+
- !ruby/object:Gem::Dependency
|
|
49
|
+
name: rspec
|
|
50
|
+
prerelease: false
|
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
|
52
|
+
requirements:
|
|
53
|
+
- - ">="
|
|
54
|
+
- !ruby/object:Gem::Version
|
|
55
|
+
segments:
|
|
56
|
+
- 1
|
|
57
|
+
- 3
|
|
58
|
+
- 0
|
|
59
|
+
version: 1.3.0
|
|
60
|
+
type: :development
|
|
61
|
+
version_requirements: *id003
|
|
62
|
+
description: Fork of TwitterStream by Vladimir Koklesnikov. Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only.
|
|
63
|
+
email: eric@xhfive.com
|
|
64
|
+
executables: []
|
|
65
|
+
|
|
66
|
+
extensions: []
|
|
67
|
+
|
|
68
|
+
extra_rdoc_files:
|
|
69
|
+
- README.markdown
|
|
70
|
+
files:
|
|
71
|
+
- .gitignore
|
|
72
|
+
- README.markdown
|
|
73
|
+
- Rakefile
|
|
74
|
+
- VERSION
|
|
75
|
+
- examples/reader.rb
|
|
76
|
+
- fixtures/twitter_stream/basic_http.txt
|
|
77
|
+
- lib/twitter_stream/.json_stream.rb.swp
|
|
78
|
+
- lib/twitter_stream/json_stream.rb
|
|
79
|
+
- spec/spec_helper.rb
|
|
80
|
+
- spec/twitter_stream/json_stream.rb
|
|
81
|
+
- xh5-twitter-stream.gemspec
|
|
82
|
+
has_rdoc: true
|
|
83
|
+
homepage: http://github.com/erichurst/twitter-stream
|
|
84
|
+
licenses: []
|
|
85
|
+
|
|
86
|
+
post_install_message:
|
|
87
|
+
rdoc_options:
|
|
88
|
+
- --charset=UTF-8
|
|
89
|
+
require_paths:
|
|
90
|
+
- lib
|
|
91
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
92
|
+
requirements:
|
|
93
|
+
- - ">="
|
|
94
|
+
- !ruby/object:Gem::Version
|
|
95
|
+
segments:
|
|
96
|
+
- 0
|
|
97
|
+
version: "0"
|
|
98
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
99
|
+
requirements:
|
|
100
|
+
- - ">="
|
|
101
|
+
- !ruby/object:Gem::Version
|
|
102
|
+
segments:
|
|
103
|
+
- 0
|
|
104
|
+
version: "0"
|
|
105
|
+
requirements: []
|
|
106
|
+
|
|
107
|
+
rubyforge_project:
|
|
108
|
+
rubygems_version: 1.3.6
|
|
109
|
+
signing_key:
|
|
110
|
+
specification_version: 3
|
|
111
|
+
summary: Twitter realtime API client
|
|
112
|
+
test_files:
|
|
113
|
+
- spec/spec_helper.rb
|
|
114
|
+
- spec/twitter_stream/json_stream.rb
|
|
115
|
+
- examples/reader.rb
|