twitter-stream 0.1.4 → 0.1.6

data/Rakefile

@@ -7,6 +7,7 @@ Spec::Rake::SpecTask.new('spec') do |t|
   t.spec_files = FileList['spec/**/*.rb']
   t.spec_opts = %w(-fs --color)
 end
+task :default => :spec
 
 begin
   require 'jeweler'

data/VERSION

@@ -1 +1 @@
-0.1.4
+0.1.6

data/lib/twitter/json_stream.rb

@@ -1,9 +1,10 @@
 require 'eventmachine'
 require 'em/buftok'
+require 'uri'
 
 module Twitter
   class JSONStream < EventMachine::Connection
-    MAX_LINE_LENGTH = 16*1024
+    MAX_LINE_LENGTH = 1024*1024
     
     # network failure reconnections
     NF_RECONNECT_START = 0.25 
@@ -25,8 +26,11 @@ module Twitter
       :path         => '/1/statuses/filter.json',
       :host         => 'stream.twitter.com',
       :port         => 80,
+      :ssl          => false,
       :auth         => 'test:test',
       :user_agent   => 'TwitterStream',
+      :timeout      => 0,
+      :proxy        => ENV['HTTP_PROXY']
     }
 
     attr_accessor :code
@@ -34,10 +38,24 @@ module Twitter
     attr_accessor :nf_last_reconnect
     attr_accessor :af_last_reconnect
     attr_accessor :reconnect_retries
+    attr_accessor :proxy
     
     def self.connect options = {}
+      options[:port] = 443 if options[:ssl] && !options.has_key?(:port)
       options = DEFAULT_OPTIONS.merge(options)
-      EventMachine.connect options[:host], options[:port], self, options
+
+      host = options[:host]
+      port = options[:port]
+
+      if options[:proxy]
+        proxy_uri = URI.parse(options[:proxy])
+        host = proxy_uri.host
+        port = proxy_uri.port
+      end
+      
+      connection = EventMachine.connect host, port, self, options
+      connection.start_tls if options[:ssl]
+      connection
     end
 
     def initialize options = {}
@@ -46,6 +64,8 @@ module Twitter
       @nf_last_reconnect = nil
       @af_last_reconnect = nil
       @reconnect_retries = 0
+      @immediate_reconnect = false
+      @proxy = URI.parse(options[:proxy]) if options[:proxy]
     end
 
     def each_item &block
@@ -68,6 +88,12 @@ module Twitter
       @gracefully_closed = true
       close_connection
     end
+
+    def immediate_reconnect
+      @immediate_reconnect = true
+      @gracefully_closed = false
+      close_connection
+    end
     
     def unbind
       receive_line(@buffer.flush) unless @buffer.empty?
@@ -80,7 +106,7 @@ module Twitter
           receive_line(line)
         end
       rescue Exception => e
-        receive_error(e.message)
+        receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
         close_connection
         return
       end
@@ -104,12 +130,22 @@ module Twitter
     
     def reconnect_after timeout
       @reconnect_callback.call(timeout, @reconnect_retries) if @reconnect_callback
-      EventMachine.add_timer(timeout) do
+
+      if timeout == 0
         reconnect @options[:host], @options[:port]
+      else
+        EventMachine.add_timer(timeout) do
+          reconnect @options[:host], @options[:port]
+        end
       end
     end
     
     def reconnect_timeout
+      if @immediate_reconnect
+        @immediate_reconnect = false
+        return 0
+      end
+      
       if (@code == 0) # network failure
         if @nf_last_reconnect
           @nf_last_reconnect += NF_RECONNECT_ADD
@@ -128,6 +164,7 @@ module Twitter
     end
   
     def reset_state
+      set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
       @code    = 0
       @headers = []
       @state   = :init
@@ -136,10 +173,18 @@ module Twitter
 
     def send_request
       data = []
-      data << "#{@options[:method]} #{@options[:path]} HTTP/1.1"
+      request_uri = @options[:path]
+      if @proxy
+        # proxies need the request to be for the full url
+        request_uri = "http#{'s' if @options[:ssl]}://#{@options[:host]}:#{@options[:port]}#{request_uri}"
+      end
+      data << "#{@options[:method]} #{request_uri} HTTP/1.1"
       data << "Host: #{@options[:host]}"
       data << "User-agent: #{@options[:user_agent]}" if @options[:user_agent]
       data << "Authorization: Basic " + [@options[:auth]].pack('m').delete("\r\n")
+      if @proxy && @proxy.user
+        data << "Proxy-Authorization: Basic " + ["#{@proxy.user}:#{@proxy.password}"].pack('m').delete("\r\n")
+      end
       if @options[:method] == 'POST'
         data << "Content-type: #{@options[:content_type]}"
         data << "Content-length: #{@options[:content].length}"

data/spec/spec_helper.rb

@@ -12,4 +12,15 @@ end
 
 def read_fixture(path)
   File.read(fixture_path(path))
+end
+
+def connect_stream(opts={}, &blk)
+  EM.run {
+    opts.merge!(:host => Host, :port => Port)
+    stop_in = opts.delete(:stop_in) || 0.5
+    EM.start_server Host, Port, JSONServer
+    @stream = JSONStream.connect(opts)
+    blk.call if blk
+    EM.add_timer(stop_in){ EM.stop }
+  }
 end

data/spec/twitter/json_stream.rb

@@ -1,3 +1,4 @@
+$:.unshift "."
 require File.dirname(__FILE__) + '/../spec_helper.rb'
 require 'twitter/json_stream'
 
@@ -23,9 +24,18 @@ describe JSONStream do
       stream = JSONStream.connect {}
     end
     
+    it "should connect to the proxy if provided" do
+      EM.should_receive(:connect).with do |host, port, handler, opts|
+        host.should == 'my-proxy'
+        port.should == 8080
+        opts[:host].should == 'stream.twitter.com'
+        opts[:port].should == 80
+        opts[:proxy].should == 'http://my-proxy:8080'
+      end
+      stream = JSONStream.connect(:proxy => "http://my-proxy:8080") {}
+    end
   end
   
-  
   Host = "127.0.0.1"
   Port = 9550
   
@@ -41,6 +51,7 @@ describe JSONStream do
   end
   
   context "on valid stream" do
+    attr_reader :stream
     before :each do
       $data_to_send = read_fixture('twitter/basic_http.txt')
       $recieved_data = ''
@@ -48,142 +59,104 @@ describe JSONStream do
     end
     
     it "should parse headers" do
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        EM.add_timer(0.5){ EM.stop }
-      }
-      @stream.code.should == 200
-      @stream.headers[0].downcase.should include 'content-type'
+      connect_stream
+      stream.code.should == 200
+      stream.headers[0].downcase.should include 'content-type'
     end
     
     it "should parse headers even after connection close" do
-      EM.run {
-        $close_connection = true
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        EM.add_timer(0.5){ EM.stop }
-      }
-      @stream.code.should == 200
-      @stream.headers[0].downcase.should include 'content-type'
+      connect_stream
+      stream.code.should == 200
+      stream.headers[0].downcase.should include 'content-type'
     end
     
     it "should extract records" do
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port, :user_agent => 'TEST_USER_AGENT'
-        EM.add_timer(0.5){ EM.stop }
-      }
+      connect_stream :user_agent => 'TEST_USER_AGENT'
       $recieved_data.upcase.should include('USER-AGENT: TEST_USER_AGENT')
     end
     
     it "should send correct user agent" do
-      EM.run {
-        $close_connection = true
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        EM.add_timer(0.5){ EM.stop }
-      }
+      connect_stream
     end
   end
   
   context "on network failure" do
+    attr_reader :stream
     before :each do
       $data_to_send = ''
       $close_connection = true
     end
     
+    it "should timeout on inactivity" do
+      connect_stream :stop_in => 1.5 do
+        stream.should_receive(:reconnect)        
+      end
+    end    
+    
     it "should reconnect on network failure" do
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        @stream.should_receive(:reconnect)
-        EM.add_timer(0.5){ EM.stop }
-      }
+      connect_stream do
+        stream.should_receive(:reconnect)
+      end
     end
     
     it "should reconnect with 0.25 at base" do
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        @stream.should_receive(:reconnect_after).with(0.25)
-        EM.add_timer(0.5){ EM.stop }
-      }
+      connect_stream do
+        stream.should_receive(:reconnect_after).with(0.25)
+      end
     end
     
     it "should reconnect with linear timeout" do
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        @stream.nf_last_reconnect = 1
-        @stream.should_receive(:reconnect_after).with(1.25)
-        EM.add_timer(0.5){ EM.stop }
-      }
+      connect_stream do
+        stream.nf_last_reconnect = 1
+        stream.should_receive(:reconnect_after).with(1.25)
+      end
     end
     
     it "should stop reconnecting after 100 times" do
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        @stream.reconnect_retries = 100
-        @stream.should_not_receive(:reconnect_after)
-        EM.add_timer(0.5){ EM.stop }
-      }
+      connect_stream do
+        stream.reconnect_retries = 100
+        stream.should_not_receive(:reconnect_after)
+      end
     end
     
     it "should notify after reconnect limit is reached" do
       timeout, retries = nil, nil
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        @stream.on_max_reconnects do |t, r|
+      connect_stream do
+        stream.on_max_reconnects do |t, r|
           timeout, retries = t, r
         end
-        @stream.reconnect_retries = 100
-        EM.add_timer(0.5){ EM.stop }
-      }
+        stream.reconnect_retries = 100
+      end
       timeout.should == 0.25
       retries.should == 101
     end
   end
   
   context "on application failure" do
+    attr_reader :stream
     before :each do
       $data_to_send = 'HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Basic realm="Firehose"\r\n\r\n1'
       $close_connection = true
     end
     
     it "should reconnect on application failure 10 at base" do
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        @stream.should_receive(:reconnect_after).with(10)
-        EM.add_timer(0.5){ EM.stop }
-      }
+      connect_stream do
+        stream.should_receive(:reconnect_after).with(10)
+      end
     end
     
     it "should reconnect with exponential timeout" do
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        @stream.af_last_reconnect = 160
-        @stream.should_receive(:reconnect_after).with(320)
-        EM.add_timer(0.5){ EM.stop }
-      }
+      connect_stream do
+        stream.af_last_reconnect = 160
+        stream.should_receive(:reconnect_after).with(320)
+      end
     end
     
     it "should not try to reconnect after limit is reached" do
-      EM.run {
-        EM.start_server Host, Port, JSONServer
-        @stream = JSONStream.connect :host => Host, :port => Port
-        @stream.af_last_reconnect = 320
-        @stream.should_not_receive(:reconnect_after)
-        EM.add_timer(0.5){ EM.stop }
-      }
+      connect_stream do
+        stream.af_last_reconnect = 320
+        stream.should_not_receive(:reconnect_after)
+      end
     end
-  end
-  
-  
-  
-end
-
+  end  
+end

data/twitter-stream.gemspec

@@ -1,12 +1,15 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{twitter-stream}
-  s.version = "0.1.4"
+  s.version = "0.1.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Vladimir Kolesnikov"]
-  s.date = %q{2009-12-01}
+  s.date = %q{2010-05-19}
   s.description = %q{Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only.}
   s.email = %q{voloko@gmail.com}
   s.extra_rdoc_files = [
@@ -27,7 +30,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/voloko/twitter-stream}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.5}
+  s.rubygems_version = %q{1.3.6}
   s.summary = %q{Twitter realtime API client}
   s.test_files = [
     "spec/spec_helper.rb",
@@ -51,3 +54,4 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<rspec>, [">= 1.2.8"])
   end
 end
+

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: twitter-stream
 version: !ruby/object:Gem::Version 
-  version: 0.1.4
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 6
+  version: 0.1.6
 platform: ruby
 authors: 
 - Vladimir Kolesnikov
@@ -9,29 +14,37 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-01 00:00:00 +03:00
+date: 2010-05-19 00:00:00 +04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: eventmachine
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 12
+        - 8
         version: 0.12.8
-    version: 
+  type: :runtime
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: rspec
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 2
+        - 8
         version: 1.2.8
-    version: 
+  type: :development
+  version_requirements: *id002
 description: Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only.
 email: voloko@gmail.com
 executables: []
@@ -64,18 +77,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Twitter realtime API client