lukemelia-twitter-stream 0.1.15

Sign up to get free protection for your applications and to get access to all the features.
data/.gemtest ADDED
File without changes
data/.gitignore ADDED
@@ -0,0 +1,10 @@
1
+ *.sw?
2
+ .rake_tasks~
3
+ .DS_Store
4
+ coverage
5
+ rdoc
6
+ pkg
7
+ Gemfile.lock
8
+ coverage/*
9
+ .yardoc/*
10
+ doc/*
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --colour
2
+ --format doc
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source :rubygems
2
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2012 Vladimir Kolesnikov, twitter-stream
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.markdown ADDED
@@ -0,0 +1,42 @@
1
+ # twitter-stream
2
+
3
+ Simple Ruby client library for [twitter streaming API](http://apiwiki.twitter.com/Streaming-API-Documentation).
4
+ Uses [EventMachine](http://rubyeventmachine.com/) for connection handling. Adheres to twitter's [reconnection guidline](https://dev.twitter.com/docs/streaming-api/concepts#connecting).
5
+
6
+ JSON format only.
7
+
8
+ ## Install
9
+
10
+ sudo gem install twitter-stream -s http://gemcutter.org
11
+
12
+ ## Usage
13
+
14
+ require 'rubygems'
15
+ require 'twitter/json_stream'
16
+
17
+ EventMachine::run {
18
+ stream = Twitter::JSONStream.connect(
19
+ :path => '/1/statuses/filter.json?track=football',
20
+ :auth => 'LOGIN:PASSWORD'
21
+ )
22
+
23
+ stream.each_item do |item|
24
+ # Do someting with unparsed JSON item.
25
+ end
26
+
27
+ stream.on_error do |message|
28
+ # No need to worry here. It might be an issue with Twitter.
29
+ # Log message for future reference. JSONStream will try to reconnect after a timeout.
30
+ end
31
+
32
+ stream.on_max_reconnects do |timeout, retries|
33
+ # Something is wrong on your side. Send yourself an email.
34
+ end
35
+ }
36
+
37
+
38
+ ## Examples
39
+
40
+ Open examples/reader.rb. Replace LOGIN:PASSWORD with your real twitter login and password. And
41
+ ruby examples/reader.rb
42
+
data/Rakefile ADDED
@@ -0,0 +1,12 @@
1
+ require 'rubygems'
2
+
3
+ require 'bundler'
4
+ Bundler::GemHelper.install_tasks
5
+
6
+ gem 'rspec', '>= 2.5.0'
7
+ require 'rspec/core/rake_task'
8
+
9
+ desc "Run all specs"
10
+ RSpec::Core::RakeTask.new(:spec)
11
+ task :default => :spec
12
+ task :test => :spec
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.14
@@ -0,0 +1,40 @@
1
+ require 'rubygems'
2
+ lib_path = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
3
+ $LOAD_PATH.unshift lib_path unless $LOAD_PATH.include?(lib_path)
4
+
5
+ require 'twitter/json_stream'
6
+
7
+ EventMachine::run {
8
+ stream = Twitter::JSONStream.connect(
9
+ :path => '/1/statuses/filter.json',
10
+ :auth => 'LOGIN:PASSWORD',
11
+ :method => 'POST',
12
+ :content => 'track=basketball,football,baseball,footy,soccer'
13
+ )
14
+
15
+ stream.each_item do |item|
16
+ $stdout.print "item: #{item}\n"
17
+ $stdout.flush
18
+ end
19
+
20
+ stream.on_error do |message|
21
+ $stdout.print "error: #{message}\n"
22
+ $stdout.flush
23
+ end
24
+
25
+ stream.on_reconnect do |timeout, retries|
26
+ $stdout.print "reconnecting in: #{timeout} seconds\n"
27
+ $stdout.flush
28
+ end
29
+
30
+ stream.on_max_reconnects do |timeout, retries|
31
+ $stdout.print "Failed after #{retries} failed reconnects\n"
32
+ $stdout.flush
33
+ end
34
+
35
+ trap('TERM') {
36
+ stream.stop
37
+ EventMachine.stop if EventMachine.reactor_running?
38
+ }
39
+ }
40
+ puts "The event loop has ended"
@@ -0,0 +1,4 @@
1
+ {"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world! Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}
2
+ {"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch: http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}
3
+ {"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats. http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}
4
+ {"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}
@@ -0,0 +1,344 @@
1
+ require 'eventmachine'
2
+ require 'em/buftok'
3
+ require 'uri'
4
+ require 'simple_oauth'
5
+ require 'http/parser'
6
+
7
+ module Twitter
8
+ class JSONStream < EventMachine::Connection
9
+ MAX_LINE_LENGTH = 1024*1024
10
+
11
+ # network failure reconnections
12
+ NF_RECONNECT_START = 0.25
13
+ NF_RECONNECT_ADD = 0.25
14
+ NF_RECONNECT_MAX = 16
15
+
16
+ # app failure reconnections
17
+ AF_RECONNECT_START = 10
18
+ AF_RECONNECT_MUL = 2
19
+
20
+ RECONNECT_MAX = 320
21
+ RETRIES_MAX = 10
22
+
23
+ DEFAULT_OPTIONS = {
24
+ :method => 'GET',
25
+ :path => '/',
26
+ :content_type => "application/x-www-form-urlencoded",
27
+ :content => '',
28
+ :path => '/1/statuses/filter.json',
29
+ :host => 'stream.twitter.com',
30
+ :port => 443,
31
+ :ssl => true,
32
+ :user_agent => 'TwitterStream',
33
+ :timeout => 0,
34
+ :proxy => ENV['HTTP_PROXY'],
35
+ :auth => nil,
36
+ :oauth => {},
37
+ :filters => [],
38
+ :params => {},
39
+ :auto_reconnect => true
40
+ }
41
+
42
+ attr_accessor :code
43
+ attr_accessor :headers
44
+ attr_accessor :nf_last_reconnect
45
+ attr_accessor :af_last_reconnect
46
+ attr_accessor :reconnect_retries
47
+ attr_accessor :proxy
48
+
49
+ def self.connect options = {}
50
+ options[:port] = 443 if options[:ssl] && !options.has_key?(:port)
51
+ options = DEFAULT_OPTIONS.merge(options)
52
+
53
+ host = options[:host]
54
+ port = options[:port]
55
+
56
+ if options[:proxy]
57
+ proxy_uri = URI.parse(options[:proxy])
58
+ host = proxy_uri.host
59
+ port = proxy_uri.port
60
+ end
61
+
62
+ connection = EventMachine.connect host, port, self, options
63
+ connection
64
+ end
65
+
66
+ def initialize options = {}
67
+ @options = DEFAULT_OPTIONS.merge(options) # merge in case initialize called directly
68
+ @gracefully_closed = false
69
+ @nf_last_reconnect = nil
70
+ @af_last_reconnect = nil
71
+ @reconnect_retries = 0
72
+ @immediate_reconnect = false
73
+ @on_inited_callback = options.delete(:on_inited)
74
+ @proxy = URI.parse(options[:proxy]) if options[:proxy]
75
+ end
76
+
77
+ def each_item &block
78
+ @each_item_callback = block
79
+ end
80
+
81
+ def on_error &block
82
+ @error_callback = block
83
+ end
84
+
85
+ def on_reconnect &block
86
+ @reconnect_callback = block
87
+ end
88
+
89
+ def on_max_reconnects &block
90
+ @max_reconnects_callback = block
91
+ end
92
+
93
+ def on_close &block
94
+ @close_callback = block
95
+ end
96
+
97
+ def stop
98
+ @gracefully_closed = true
99
+ close_connection
100
+ end
101
+
102
+ def immediate_reconnect
103
+ @immediate_reconnect = true
104
+ @gracefully_closed = false
105
+ close_connection
106
+ end
107
+
108
+ def unbind
109
+ if @state == :stream && !@buffer.empty?
110
+ parse_stream_line(@buffer.flush)
111
+ end
112
+ schedule_reconnect if @options[:auto_reconnect] && !@gracefully_closed
113
+ @close_callback.call if @close_callback
114
+
115
+ end
116
+
117
+ # Receives raw data from the HTTP connection and pushes it into the
118
+ # HTTP parser which then drives subsequent callbacks.
119
+ def receive_data(data)
120
+ @parser << data
121
+ end
122
+
123
+ def connection_completed
124
+ start_tls if @options[:ssl]
125
+ send_request
126
+ end
127
+
128
+ def post_init
129
+ reset_state
130
+ @on_inited_callback.call if @on_inited_callback
131
+ end
132
+
133
+ protected
134
+ def schedule_reconnect
135
+ timeout = reconnect_timeout
136
+ @reconnect_retries += 1
137
+ if timeout <= RECONNECT_MAX && @reconnect_retries <= RETRIES_MAX
138
+ reconnect_after(timeout)
139
+ else
140
+ @max_reconnects_callback.call(timeout, @reconnect_retries) if @max_reconnects_callback
141
+ end
142
+ end
143
+
144
+ def reconnect_after timeout
145
+ @reconnect_callback.call(timeout, @reconnect_retries) if @reconnect_callback
146
+
147
+ if timeout == 0
148
+ reconnect @options[:host], @options[:port]
149
+ else
150
+ EventMachine.add_timer(timeout) do
151
+ reconnect @options[:host], @options[:port]
152
+ end
153
+ end
154
+ end
155
+
156
+ def reconnect_timeout
157
+ if @immediate_reconnect
158
+ @immediate_reconnect = false
159
+ return 0
160
+ end
161
+
162
+ if (@code == 0) # network failure
163
+ if @nf_last_reconnect
164
+ @nf_last_reconnect += NF_RECONNECT_ADD
165
+ else
166
+ @nf_last_reconnect = NF_RECONNECT_START
167
+ end
168
+ [@nf_last_reconnect,NF_RECONNECT_MAX].min
169
+ else
170
+ if @af_last_reconnect
171
+ @af_last_reconnect *= AF_RECONNECT_MUL
172
+ else
173
+ @af_last_reconnect = AF_RECONNECT_START
174
+ end
175
+ @af_last_reconnect
176
+ end
177
+ end
178
+
179
+ def reset_state
180
+ set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
181
+ @code = 0
182
+ @headers = {}
183
+ @state = :init
184
+ @buffer = BufferedTokenizer.new("\r", MAX_LINE_LENGTH)
185
+ @stream = ''
186
+
187
+ @parser = Http::Parser.new
188
+ @parser.on_headers_complete = method(:handle_headers_complete)
189
+ @parser.on_body = method(:receive_stream_data)
190
+ end
191
+
192
+ # Called when the status line and all headers have been read from the
193
+ # stream.
194
+ def handle_headers_complete(headers)
195
+ @code = @parser.status_code.to_i
196
+ if @code != 200
197
+ receive_error("invalid status code: #{@code}.")
198
+ end
199
+ self.headers = headers
200
+ @state = :stream
201
+ end
202
+
203
+ # Called every time a chunk of data is read from the connection once it has
204
+ # been opened and after the headers have been processed.
205
+ def receive_stream_data(data)
206
+ begin
207
+ @buffer.extract(data).each do |line|
208
+ parse_stream_line(line)
209
+ end
210
+ @stream = ''
211
+ rescue Exception => e
212
+ receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
213
+ close_connection
214
+ return
215
+ end
216
+ end
217
+
218
+ def send_request
219
+ data = []
220
+ request_uri = @options[:path]
221
+
222
+ if @proxy
223
+ # proxies need the request to be for the full url
224
+ request_uri = "#{uri_base}:#{@options[:port]}#{request_uri}"
225
+ end
226
+
227
+ content = @options[:content]
228
+
229
+ unless (q = query).empty?
230
+ if @options[:method].to_s.upcase == 'GET'
231
+ request_uri << "?#{q}"
232
+ else
233
+ content = q
234
+ end
235
+ end
236
+
237
+ data << "#{@options[:method]} #{request_uri} HTTP/1.1"
238
+ data << "Host: #{@options[:host]}"
239
+ data << 'Accept: */*'
240
+ data << "User-Agent: #{@options[:user_agent]}" if @options[:user_agent]
241
+
242
+ if @options[:auth]
243
+ data << "Authorization: Basic #{[@options[:auth]].pack('m').delete("\r\n")}"
244
+ elsif @options[:oauth]
245
+ data << "Authorization: #{oauth_header}"
246
+ end
247
+
248
+ if @proxy && @proxy.user
249
+ data << "Proxy-Authorization: Basic " + ["#{@proxy.user}:#{@proxy.password}"].pack('m').delete("\r\n")
250
+ end
251
+ if ['POST', 'PUT'].include?(@options[:method])
252
+ data << "Content-type: #{@options[:content_type]}"
253
+ data << "Content-length: #{content.length}"
254
+ end
255
+
256
+ if @options[:headers]
257
+ @options[:headers].each do |name,value|
258
+ data << "#{name}: #{value}"
259
+ end
260
+ end
261
+
262
+ data << "\r\n"
263
+
264
+ send_data data.join("\r\n") << content
265
+ end
266
+
267
+ def receive_error e
268
+ @error_callback.call(e) if @error_callback
269
+ end
270
+
271
+ def parse_stream_line ln
272
+ ln.strip!
273
+ unless ln.empty?
274
+ if ln[0,1] == '{' || ln[ln.length-1,1] == '}'
275
+ @stream << ln
276
+ if @stream[0,1] == '{' && @stream[@stream.length-1,1] == '}'
277
+ @each_item_callback.call(@stream) if @each_item_callback
278
+ @stream = ''
279
+ end
280
+ end
281
+ end
282
+ end
283
+
284
+ def reset_timeouts
285
+ set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
286
+ @nf_last_reconnect = @af_last_reconnect = nil
287
+ @reconnect_retries = 0
288
+ end
289
+
290
+ #
291
+ # URL and request components
292
+ #
293
+
294
+ # :filters => %w(miama lebron jesus)
295
+ # :oauth => {
296
+ # :consumer_key => [key],
297
+ # :consumer_secret => [token],
298
+ # :access_key => [access key],
299
+ # :access_secret => [access secret]
300
+ # }
301
+ def oauth_header
302
+ uri = uri_base + @options[:path].to_s
303
+
304
+ # The hash SimpleOAuth accepts is slightly different from that of
305
+ # ROAuth. To preserve backward compatability, fix the cache here
306
+ # so that the arguments passed in don't need to change.
307
+ oauth = {
308
+ :consumer_key => @options[:oauth][:consumer_key],
309
+ :consumer_secret => @options[:oauth][:consumer_secret],
310
+ :token => @options[:oauth][:access_key],
311
+ :token_secret => @options[:oauth][:access_secret]
312
+ }
313
+
314
+ data = ['POST', 'PUT'].include?(@options[:method]) ? params : {}
315
+
316
+ SimpleOAuth::Header.new(@options[:method], uri, data, oauth)
317
+ end
318
+
319
+ # Scheme (https if ssl, http otherwise) and host part of URL
320
+ def uri_base
321
+ "http#{'s' if @options[:ssl]}://#{@options[:host]}"
322
+ end
323
+
324
+ # Normalized query hash of escaped string keys and escaped string values
325
+ # nil values are skipped
326
+ def params
327
+ flat = {}
328
+ @options[:params].merge( :track => @options[:filters] ).each do |param, val|
329
+ next if val.to_s.empty? || (val.respond_to?(:empty?) && val.empty?)
330
+ val = val.join(",") if val.respond_to?(:join)
331
+ flat[param.to_s] = val.to_s
332
+ end
333
+ flat
334
+ end
335
+
336
+ def query
337
+ params.map{|param, value| [escape(param), escape(value)].join("=")}.sort.join("&")
338
+ end
339
+
340
+ def escape str
341
+ URI.escape(str.to_s, /[^a-zA-Z0-9\-\.\_\~]/)
342
+ end
343
+ end
344
+ end
@@ -0,0 +1,32 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ Gem::Specification.new do |s|
4
+ s.name = %q{lukemelia-twitter-stream}
5
+ s.version = "0.1.15"
6
+
7
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
8
+ s.authors = ["Vladimir Kolesnikov"]
9
+ s.date = %q{2012-04-10}
10
+ s.description = %q{Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only. (Fork to update eventmachine dependency)}
11
+ s.summary = %q{Twitter realtime API client}
12
+ s.homepage = %q{http://github.com/lukemelia/twitter-stream}
13
+ s.email = %q{voloko@gmail.com}
14
+
15
+ s.platform = Gem::Platform::RUBY
16
+ s.rubygems_version = %q{1.3.6}
17
+ s.required_rubygems_version = Gem::Requirement.new(">= 1.3.6") if s.respond_to? :required_rubygems_version=
18
+
19
+ s.rdoc_options = ["--charset=UTF-8"]
20
+ s.extra_rdoc_files = ["README.markdown", "LICENSE"]
21
+
22
+ s.add_runtime_dependency('eventmachine', ">= 1.0.0.beta.4")
23
+ s.add_runtime_dependency('simple_oauth', '~> 0.1.4')
24
+ s.add_runtime_dependency('http_parser.rb', '~> 0.5.1')
25
+ s.add_development_dependency('rspec', "~> 2.5.0")
26
+
27
+ s.files = `git ls-files`.split("\n")
28
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
29
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
30
+ s.require_paths = ["lib"]
31
+ end
32
+
@@ -0,0 +1,53 @@
1
+ require 'rubygems'
2
+
3
+ gem 'rspec', '>= 2.5.0'
4
+ require 'rspec'
5
+ require 'rspec/mocks'
6
+
7
+ require 'twitter/json_stream'
8
+
9
+ def fixture_path(path)
10
+ File.join(File.dirname(__FILE__), '..', 'fixtures', path)
11
+ end
12
+
13
+ def read_fixture(path)
14
+ File.read(fixture_path(path))
15
+ end
16
+
17
+ def connect_stream(opts={}, &blk)
18
+ EM.run {
19
+ opts.merge!(:host => Host, :port => Port)
20
+ stop_in = opts.delete(:stop_in) || 0.5
21
+ unless opts[:start_server] == false
22
+ EM.start_server Host, Port, JSONServer
23
+ end
24
+ @stream = JSONStream.connect(opts)
25
+ blk.call if blk
26
+ EM.add_timer(stop_in){ EM.stop }
27
+ }
28
+ end
29
+
30
+ def http_response(status_code, status_text, headers, body)
31
+ res = "HTTP/1.1 #{status_code} #{status_text}\r\n"
32
+ headers = {
33
+ "Content-Type"=>"application/json",
34
+ "Transfer-Encoding"=>"chunked"
35
+ }.merge(headers)
36
+ headers.each do |key,value|
37
+ res << "#{key}: #{value}\r\n"
38
+ end
39
+ res << "\r\n"
40
+ if headers["Transfer-Encoding"] == "chunked" && body.kind_of?(Array)
41
+ body.each do |data|
42
+ res << http_chunk(data)
43
+ end
44
+ else
45
+ res << body
46
+ end
47
+ res
48
+ end
49
+
50
+ def http_chunk(data)
51
+ # See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
52
+ "#{data.length.to_s(16)}\r\n#{data}\r\n"
53
+ end
@@ -0,0 +1,302 @@
1
+ require 'spec_helper.rb'
2
+ require 'twitter/json_stream'
3
+
4
+ include Twitter
5
+
6
+ Host = "127.0.0.1"
7
+ Port = 9550
8
+
9
+ class JSONServer < EM::Connection
10
+ attr_accessor :data
11
+ def receive_data data
12
+ $recieved_data = data
13
+ send_data $data_to_send
14
+ EventMachine.next_tick {
15
+ close_connection if $close_connection
16
+ }
17
+ end
18
+ end
19
+
20
+
21
+
22
+ describe JSONStream do
23
+
24
+ context "authentication" do
25
+ it "should connect with basic auth credentials" do
26
+ connect_stream :auth => "username:password", :ssl => false
27
+ $recieved_data.should include('Authorization: Basic')
28
+ end
29
+
30
+ it "should connect with oauth credentials" do
31
+ oauth = {
32
+ :consumer_key => '1234567890',
33
+ :consumer_secret => 'abcdefghijklmnopqrstuvwxyz',
34
+ :access_key => 'ohai',
35
+ :access_secret => 'ohno'
36
+ }
37
+ connect_stream :oauth => oauth, :ssl => false
38
+ $recieved_data.should include('Authorization: OAuth')
39
+ end
40
+ end
41
+
42
+ context "on create" do
43
+
44
+ it "should return stream" do
45
+ EM.should_receive(:connect).and_return('TEST INSTANCE')
46
+ stream = JSONStream.connect {}
47
+ stream.should == 'TEST INSTANCE'
48
+ end
49
+
50
+ it "should define default properties" do
51
+ EM.should_receive(:connect).with do |host, port, handler, opts|
52
+ host.should == 'stream.twitter.com'
53
+ port.should == 443
54
+ opts[:path].should == '/1/statuses/filter.json'
55
+ opts[:method].should == 'GET'
56
+ end
57
+ stream = JSONStream.connect {}
58
+ end
59
+
60
+ it "should connect to the proxy if provided" do
61
+ EM.should_receive(:connect).with do |host, port, handler, opts|
62
+ host.should == 'my-proxy'
63
+ port.should == 8080
64
+ opts[:host].should == 'stream.twitter.com'
65
+ opts[:port].should == 443
66
+ opts[:proxy].should == 'http://my-proxy:8080'
67
+ end
68
+ stream = JSONStream.connect(:proxy => "http://my-proxy:8080") {}
69
+ end
70
+
71
+ it "should not trigger SSL until connection is established" do
72
+ connection = stub('connection')
73
+ EM.should_receive(:connect).and_return(connection)
74
+ stream = JSONStream.connect(:ssl => true)
75
+ stream.should == connection
76
+ end
77
+
78
+ end
79
+
80
+ context "on valid stream" do
81
+ attr_reader :stream
82
+ before :each do
83
+ $body = File.readlines(fixture_path("twitter/tweets.txt"))
84
+ $body.each {|tweet| tweet.strip!; tweet << "\r" }
85
+ $data_to_send = http_response(200, "OK", {}, $body)
86
+ $recieved_data = ''
87
+ $close_connection = false
88
+ end
89
+
90
+ it "should add no params" do
91
+ connect_stream :ssl => false
92
+ $recieved_data.should include('/1/statuses/filter.json HTTP')
93
+ end
94
+
95
+ it "should add custom params" do
96
+ connect_stream :params => {:name => 'test'}, :ssl => false
97
+ $recieved_data.should include('?name=test')
98
+ end
99
+
100
+ it "should parse headers" do
101
+ connect_stream :ssl => false
102
+ stream.code.should == 200
103
+ stream.headers.keys.map{|k| k.downcase}.should include('content-type')
104
+ end
105
+
106
+ it "should parse headers even after connection close" do
107
+ connect_stream :ssl => false
108
+ stream.code.should == 200
109
+ stream.headers.keys.map{|k| k.downcase}.should include('content-type')
110
+ end
111
+
112
+ it "should extract records" do
113
+ connect_stream :user_agent => 'TEST_USER_AGENT', :ssl => false
114
+ $recieved_data.upcase.should include('USER-AGENT: TEST_USER_AGENT')
115
+ end
116
+
117
+ it 'should allow custom headers' do
118
+ connect_stream :headers => { 'From' => 'twitter-stream' }, :ssl => false
119
+ $recieved_data.upcase.should include('FROM: TWITTER-STREAM')
120
+ end
121
+
122
+ it "should deliver each item" do
123
+ items = []
124
+ connect_stream :ssl => false do
125
+ stream.each_item do |item|
126
+ items << item
127
+ end
128
+ end
129
+ # Extract only the tweets from the fixture
130
+ tweets = $body.map{|l| l.strip }.select{|l| l =~ /^\{/ }
131
+ items.size.should == tweets.size
132
+ tweets.each_with_index do |tweet,i|
133
+ items[i].should == tweet
134
+ end
135
+ end
136
+
137
+ it "should send correct user agent" do
138
+ connect_stream
139
+ end
140
+ end
141
+
142
+ shared_examples_for "network failure" do
143
+ it "should reconnect on network failure" do
144
+ connect_stream do
145
+ stream.should_receive(:reconnect)
146
+ end
147
+ end
148
+
149
+ it "should not reconnect on network failure when not configured to auto reconnect" do
150
+ connect_stream(:auto_reconnect => false) do
151
+ stream.should_receive(:reconnect).never
152
+ end
153
+ end
154
+
155
+ it "should reconnect with 0.25 at base" do
156
+ connect_stream do
157
+ stream.should_receive(:reconnect_after).with(0.25)
158
+ end
159
+ end
160
+
161
+ it "should reconnect with linear timeout" do
162
+ connect_stream do
163
+ stream.nf_last_reconnect = 1
164
+ stream.should_receive(:reconnect_after).with(1.25)
165
+ end
166
+ end
167
+
168
+ it "should stop reconnecting after 100 times" do
169
+ connect_stream do
170
+ stream.reconnect_retries = 100
171
+ stream.should_not_receive(:reconnect_after)
172
+ end
173
+ end
174
+
175
+ it "should notify after reconnect limit is reached" do
176
+ timeout, retries = nil, nil
177
+ connect_stream do
178
+ stream.on_max_reconnects do |t, r|
179
+ timeout, retries = t, r
180
+ end
181
+ stream.reconnect_retries = 100
182
+ end
183
+ timeout.should == 0.25
184
+ retries.should == 101
185
+ end
186
+ end
187
+
188
+ context "on network failure" do
189
+ attr_reader :stream
190
+ before :each do
191
+ $data_to_send = ''
192
+ $close_connection = true
193
+ end
194
+
195
+ it "should timeout on inactivity" do
196
+ connect_stream :stop_in => 1.5 do
197
+ stream.should_receive(:reconnect)
198
+ end
199
+ end
200
+
201
+ it "should not reconnect on inactivity when not configured to auto reconnect" do
202
+ connect_stream(:stop_in => 1.5, :auto_reconnect => false) do
203
+ stream.should_receive(:reconnect).never
204
+ end
205
+ end
206
+
207
+ it "should reconnect with SSL if enabled" do
208
+ connect_stream :ssl => true do
209
+ stream.should_receive(:start_tls).twice
210
+ end
211
+ end
212
+
213
+ it_should_behave_like "network failure"
214
+ end
215
+
216
+ context "on server unavailable" do
217
+
218
+ attr_reader :stream
219
+
220
+ # This is to make it so the network failure specs which call connect_stream
221
+ # can be reused. This way calls to connect_stream won't actually create a
222
+ # server to listen in.
223
+ def connect_stream_without_server(opts={},&block)
224
+ connect_stream_default(opts.merge(:start_server=>false),&block)
225
+ end
226
+ alias_method :connect_stream_default, :connect_stream
227
+ alias_method :connect_stream, :connect_stream_without_server
228
+
229
+ it_should_behave_like "network failure"
230
+ end
231
+
232
+ context "on application failure" do
233
+ attr_reader :stream
234
+ before :each do
235
+ $data_to_send = "HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Basic realm=\"Firehose\"\r\n\r\n"
236
+ $close_connection = false
237
+ end
238
+
239
+ it "should reconnect on application failure 10 at base" do
240
+ connect_stream :ssl => false do
241
+ stream.should_receive(:reconnect_after).with(10)
242
+ end
243
+ end
244
+
245
+ it "should not reconnect on application failure 10 at base when not configured to auto reconnect" do
246
+ connect_stream :ssl => false, :auto_reconnect => false do
247
+ stream.should_receive(:reconnect_after).never
248
+ end
249
+ end
250
+
251
+ it "should reconnect with exponential timeout" do
252
+ connect_stream :ssl => false do
253
+ stream.af_last_reconnect = 160
254
+ stream.should_receive(:reconnect_after).with(320)
255
+ end
256
+ end
257
+
258
+ it "should not try to reconnect after limit is reached" do
259
+ connect_stream :ssl => false do
260
+ stream.af_last_reconnect = 320
261
+ stream.should_not_receive(:reconnect_after)
262
+ end
263
+ end
264
+ end
265
+
266
+ context "on stream with chunked transfer encoding" do
267
+ attr_reader :stream
268
+ before :each do
269
+ $recieved_data = ''
270
+ $close_connection = false
271
+ end
272
+
273
+ it "should ignore empty lines" do
274
+ body_chunks = ["{\"screen"+"_name\"",":\"user1\"}\r\r\r{","\"id\":9876}\r\r"]
275
+ $data_to_send = http_response(200,"OK",{},body_chunks)
276
+ items = []
277
+ connect_stream :ssl => false do
278
+ stream.each_item do |item|
279
+ items << item
280
+ end
281
+ end
282
+ items.size.should == 2
283
+ items[0].should == '{"screen_name":"user1"}'
284
+ items[1].should == '{"id":9876}'
285
+ end
286
+
287
+ it "should parse full entities even if split" do
288
+ body_chunks = ["{\"id\"",":1234}\r{","\"id\":9876}"]
289
+ $data_to_send = http_response(200,"OK",{},body_chunks)
290
+ items = []
291
+ connect_stream :ssl => false do
292
+ stream.each_item do |item|
293
+ items << item
294
+ end
295
+ end
296
+ items.size.should == 2
297
+ items[0].should == '{"id":1234}'
298
+ items[1].should == '{"id":9876}'
299
+ end
300
+ end
301
+
302
+ end
metadata ADDED
@@ -0,0 +1,112 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: lukemelia-twitter-stream
3
+ version: !ruby/object:Gem::Version
4
+ prerelease:
5
+ version: 0.1.15
6
+ platform: ruby
7
+ authors:
8
+ - Vladimir Kolesnikov
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+
13
+ date: 2012-04-10 00:00:00 Z
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: eventmachine
17
+ prerelease: false
18
+ requirement: &id001 !ruby/object:Gem::Requirement
19
+ none: false
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: 1.0.0.beta.4
24
+ type: :runtime
25
+ version_requirements: *id001
26
+ - !ruby/object:Gem::Dependency
27
+ name: simple_oauth
28
+ prerelease: false
29
+ requirement: &id002 !ruby/object:Gem::Requirement
30
+ none: false
31
+ requirements:
32
+ - - ~>
33
+ - !ruby/object:Gem::Version
34
+ version: 0.1.4
35
+ type: :runtime
36
+ version_requirements: *id002
37
+ - !ruby/object:Gem::Dependency
38
+ name: http_parser.rb
39
+ prerelease: false
40
+ requirement: &id003 !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: 0.5.1
46
+ type: :runtime
47
+ version_requirements: *id003
48
+ - !ruby/object:Gem::Dependency
49
+ name: rspec
50
+ prerelease: false
51
+ requirement: &id004 !ruby/object:Gem::Requirement
52
+ none: false
53
+ requirements:
54
+ - - ~>
55
+ - !ruby/object:Gem::Version
56
+ version: 2.5.0
57
+ type: :development
58
+ version_requirements: *id004
59
+ description: Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only. (Fork to update eventmachine dependency)
60
+ email: voloko@gmail.com
61
+ executables: []
62
+
63
+ extensions: []
64
+
65
+ extra_rdoc_files:
66
+ - README.markdown
67
+ - LICENSE
68
+ files:
69
+ - .gemtest
70
+ - .gitignore
71
+ - .rspec
72
+ - Gemfile
73
+ - LICENSE
74
+ - README.markdown
75
+ - Rakefile
76
+ - VERSION
77
+ - examples/reader.rb
78
+ - fixtures/twitter/tweets.txt
79
+ - lib/twitter/json_stream.rb
80
+ - lukemelia-twitter-stream.gemspec
81
+ - spec/spec_helper.rb
82
+ - spec/twitter/json_stream_spec.rb
83
+ homepage: http://github.com/lukemelia/twitter-stream
84
+ licenses: []
85
+
86
+ post_install_message:
87
+ rdoc_options:
88
+ - --charset=UTF-8
89
+ require_paths:
90
+ - lib
91
+ required_ruby_version: !ruby/object:Gem::Requirement
92
+ none: false
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: "0"
97
+ required_rubygems_version: !ruby/object:Gem::Requirement
98
+ none: false
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: 1.3.6
103
+ requirements: []
104
+
105
+ rubyforge_project:
106
+ rubygems_version: 1.8.10
107
+ signing_key:
108
+ specification_version: 3
109
+ summary: Twitter realtime API client
110
+ test_files:
111
+ - spec/spec_helper.rb
112
+ - spec/twitter/json_stream_spec.rb