cantino-twitter-stream 0.1.15

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5ca1d5b33fa41468caf5c6019d5f8841db2d2e37
+  data.tar.gz: 186a3a7bd214aea7b951da75cb6737ceb93f8c3f
+SHA512:
+  metadata.gz: 1566c82d837ee63c4ead22897851c1a1d33cae3fed16c6b96cfe43549e0733c2c1d67001ed8419afd985c91212ce5ed86c3ed38476aafc83a9166f69d386c5a7
+  data.tar.gz: 894d5f39ee242f3e32a69c12399e0464148400fc69c01bb836cd65b0c99873b63212db547801b4edc8de8c88c11a5c1bb68d52fa1391bb971f847913cc78dc2a

data/.gitignore

@@ -0,0 +1,10 @@
+*.sw?
+.rake_tasks~
+.DS_Store
+coverage
+rdoc
+pkg
+Gemfile.lock
+coverage/*
+.yardoc/*
+doc/*

data/.rspec

@@ -0,0 +1,2 @@
+--colour
+--format doc

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Vladimir Kolesnikov, twitter-stream
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,47 @@
+# twitter-stream
+
+Simple Ruby client library for [twitter streaming API](http://apiwiki.twitter.com/Streaming-API-Documentation).
+Uses [EventMachine](http://rubyeventmachine.com/) for connection handling. Adheres to twitter's [reconnection guidline](https://dev.twitter.com/docs/streaming-api/concepts#connecting).
+
+JSON format only.
+
+## Install
+
+    sudo gem install twitter-stream -s http://gemcutter.org
+
+## Usage
+
+    require 'rubygems'
+    require 'twitter/json_stream'
+
+    EventMachine::run {
+      stream = Twitter::JSONStream.connect(
+        :path    => '/1/statuses/filter.json?track=football',
+        :auth    => 'LOGIN:PASSWORD'
+      )
+
+      stream.each_item do |item|
+        # Do someting with unparsed JSON item.
+      end
+
+      stream.on_error do |message|
+        # No need to worry here. It might be an issue with Twitter.
+        # Log message for future reference. JSONStream will try to reconnect after a timeout.
+      end
+
+      stream.on_max_reconnects do |timeout, retries|
+        # Something is wrong on your side. Send yourself an email.
+      end
+
+      stream.on_no_data do
+        # Twitter has stopped sending any data on the currently active
+        # connection, reconnecting is probably in order
+      end
+    }
+
+
+## Examples
+
+Open examples/reader.rb. Replace LOGIN:PASSWORD with your real twitter login and password. And
+    ruby examples/reader.rb
+

data/Rakefile

@@ -0,0 +1,12 @@
+require 'rubygems'
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+gem 'rspec', '>= 2.5.0'
+require 'rspec/core/rake_task'
+
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec
+task :test => :spec

data/VERSION

@@ -0,0 +1 @@
+0.1.14

data/examples/reader.rb

@@ -0,0 +1,40 @@
+require 'rubygems'
+lib_path = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift lib_path unless $LOAD_PATH.include?(lib_path)
+
+require 'twitter/json_stream'
+
+EventMachine::run {
+  stream = Twitter::JSONStream.connect(
+    :path    => '/1/statuses/filter.json',
+    :auth    => 'LOGIN:PASSWORD',
+    :method  => 'POST',
+    :content => 'track=basketball,football,baseball,footy,soccer'
+  )
+    
+  stream.each_item do |item|
+    $stdout.print "item: #{item}\n"
+    $stdout.flush
+  end
+  
+  stream.on_error do |message|
+    $stdout.print "error: #{message}\n"
+    $stdout.flush
+  end
+  
+  stream.on_reconnect do |timeout, retries|
+    $stdout.print "reconnecting in: #{timeout} seconds\n"
+    $stdout.flush
+  end
+  
+  stream.on_max_reconnects do |timeout, retries|
+    $stdout.print "Failed after #{retries} failed reconnects\n"
+    $stdout.flush
+  end
+  
+  trap('TERM') {  
+    stream.stop
+    EventMachine.stop if EventMachine.reactor_running? 
+  }
+}
+puts "The event loop has ended"

data/fixtures/twitter/tweets.txt

@@ -0,0 +1,4 @@
+{"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world!  Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}
+{"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch:  http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}
+{"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats.  http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}
+{"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}

data/lib/twitter/json_stream.rb

@@ -0,0 +1,366 @@
+require 'eventmachine'
+require 'em/buftok'
+require 'uri'
+require 'simple_oauth'
+require 'http/parser'
+
+module Twitter
+  class JSONStream < EventMachine::Connection
+    MAX_LINE_LENGTH = 1024*1024
+
+    # network failure reconnections
+    NF_RECONNECT_START = 0.25
+    NF_RECONNECT_ADD   = 0.25
+    NF_RECONNECT_MAX   = 16
+
+    # app failure reconnections
+    AF_RECONNECT_START = 10
+    AF_RECONNECT_MUL   = 2
+
+    RECONNECT_MAX   = 320
+    RETRIES_MAX     = 10
+
+    NO_DATA_TIMEOUT = 90
+
+    DEFAULT_OPTIONS = {
+      :method         => 'GET',
+      :path           => '/',
+      :content_type   => "application/x-www-form-urlencoded",
+      :content        => '',
+      :path           => '/1/statuses/filter.json',
+      :host           => 'stream.twitter.com',
+      :port           => 443,
+      :ssl            => true,
+      :user_agent     => 'TwitterStream',
+      :timeout        => 0,
+      :proxy          => ENV['HTTP_PROXY'],
+      :auth           => nil,
+      :oauth          => {},
+      :filters        => [],
+      :params         => {},
+      :auto_reconnect => true
+    }
+
+    attr_accessor :code
+    attr_accessor :headers
+    attr_accessor :nf_last_reconnect
+    attr_accessor :af_last_reconnect
+    attr_accessor :reconnect_retries
+    attr_accessor :last_data_received_at
+    attr_accessor :proxy
+
+    def self.connect options = {}
+      options[:port] = 443 if options[:ssl] && !options.has_key?(:port)
+      options = DEFAULT_OPTIONS.merge(options)
+
+      host = options[:host]
+      port = options[:port]
+
+      if options[:proxy]
+        proxy_uri = URI.parse(options[:proxy])
+        host = proxy_uri.host
+        port = proxy_uri.port
+      end
+
+      connection = EventMachine.connect host, port, self, options
+      connection
+    end
+
+    def initialize options = {}
+      @options = DEFAULT_OPTIONS.merge(options) # merge in case initialize called directly
+      @gracefully_closed = false
+      @nf_last_reconnect = nil
+      @af_last_reconnect = nil
+      @reconnect_retries = 0
+      @immediate_reconnect = false
+      @on_inited_callback = options.delete(:on_inited)
+      @proxy = URI.parse(options[:proxy]) if options[:proxy]
+      @last_data_received_at = nil
+    end
+
+    def each_item &block
+      @each_item_callback = block
+    end
+
+    def on_error &block
+      @error_callback = block
+    end
+
+    def on_reconnect &block
+      @reconnect_callback = block
+    end
+
+    # Called when no data has been received for NO_DATA_TIMEOUT seconds.
+    # Reconnecting is probably in order as per the Twitter recommendations
+    def on_no_data &block
+      @no_data_callback = block
+    end
+
+    def on_max_reconnects &block
+      @max_reconnects_callback = block
+    end
+
+    def on_close &block
+      @close_callback = block
+    end
+
+    def stop
+      @gracefully_closed = true
+      close_connection
+    end
+
+    def immediate_reconnect
+      @immediate_reconnect = true
+      @gracefully_closed = false
+      close_connection
+    end
+
+    def unbind
+      if @state == :stream && !@buffer.empty?
+        parse_stream_line(@buffer.flush)
+      end
+      schedule_reconnect if @options[:auto_reconnect] && !@gracefully_closed
+      @close_callback.call if @close_callback
+      @state = :init
+    end
+
+    # Receives raw data from the HTTP connection and pushes it into the
+    # HTTP parser which then drives subsequent callbacks.
+    def receive_data(data)
+      @last_data_received_at = Time.now
+      @parser << data
+    end
+
+    def connection_completed
+      start_tls if @options[:ssl]
+      send_request
+    end
+
+    def post_init
+      reset_state
+      @on_inited_callback.call if @on_inited_callback
+      @reconnect_timer = EventMachine.add_periodic_timer(5) do
+        if @gracefully_closed
+          @reconnect_timer.cancel
+        elsif @last_data_received_at && Time.now - @last_data_received_at > NO_DATA_TIMEOUT
+          no_data
+        end
+      end
+    end
+
+  protected
+    def no_data
+      @no_data_callback.call if @no_data_callback
+    end
+
+    def schedule_reconnect
+      timeout = reconnect_timeout
+      @reconnect_retries += 1
+      if timeout <= RECONNECT_MAX && @reconnect_retries <= RETRIES_MAX
+        reconnect_after(timeout)
+      else
+        @max_reconnects_callback.call(timeout, @reconnect_retries) if @max_reconnects_callback
+      end
+    end
+
+    def reconnect_after timeout
+      @reconnect_callback.call(timeout, @reconnect_retries) if @reconnect_callback
+
+      if timeout == 0
+        reconnect @options[:host], @options[:port]
+      else
+        EventMachine.add_timer(timeout) do
+          reconnect @options[:host], @options[:port]
+        end
+      end
+    end
+
+    def reconnect_timeout
+      if @immediate_reconnect
+        @immediate_reconnect = false
+        return 0
+      end
+
+      if (@code == 0) # network failure
+        if @nf_last_reconnect
+          @nf_last_reconnect += NF_RECONNECT_ADD
+        else
+          @nf_last_reconnect = NF_RECONNECT_START
+        end
+        [@nf_last_reconnect,NF_RECONNECT_MAX].min
+      else
+        if @af_last_reconnect
+          @af_last_reconnect *= AF_RECONNECT_MUL
+        else
+          @af_last_reconnect = AF_RECONNECT_START
+        end
+        @af_last_reconnect
+      end
+    end
+
+    def reset_state
+      set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
+      @code    = 0
+      @headers = {}
+      @state   = :init
+      @buffer  = BufferedTokenizer.new("\r", MAX_LINE_LENGTH)
+      @stream  = ''
+
+      @parser  = Http::Parser.new
+      @parser.on_headers_complete = method(:handle_headers_complete)
+      @parser.on_body = method(:receive_stream_data)
+    end
+
+    # Called when the status line and all headers have been read from the
+    # stream.
+    def handle_headers_complete(headers)
+      @code = @parser.status_code.to_i
+      if @code != 200
+        receive_error("invalid status code: #{@code}.")
+      end
+      self.headers = headers
+      @state = :stream
+    end
+
+    # Called every time a chunk of data is read from the connection once it has
+    # been opened and after the headers have been processed.
+    def receive_stream_data(data)
+      begin
+        @buffer.extract(data).each do |line|
+          parse_stream_line(line)
+        end
+        @stream  = ''
+      rescue => e
+        receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
+        close_connection
+        return
+      end
+    end
+
+    def send_request
+      data = []
+      request_uri = @options[:path]
+
+      if @proxy
+        # proxies need the request to be for the full url
+        request_uri = "#{uri_base}:#{@options[:port]}#{request_uri}"
+      end
+
+      content = @options[:content]
+
+      unless (q = query).empty?
+        if @options[:method].to_s.upcase == 'GET'
+          request_uri << "?#{q}"
+        else
+          content = q
+        end
+      end
+
+      data << "#{@options[:method]} #{request_uri} HTTP/1.1"
+      data << "Host: #{@options[:host]}"
+      data << 'Accept: */*'
+      data << "User-Agent: #{@options[:user_agent]}" if @options[:user_agent]
+
+      if @options[:auth]
+        data << "Authorization: Basic #{[@options[:auth]].pack('m').delete("\r\n")}"
+      elsif @options[:oauth]
+        data << "Authorization: #{oauth_header}"
+      end
+
+      if @proxy && @proxy.user
+        data << "Proxy-Authorization: Basic " + ["#{@proxy.user}:#{@proxy.password}"].pack('m').delete("\r\n")
+      end
+      if ['POST', 'PUT'].include?(@options[:method])
+        data << "Content-type: #{@options[:content_type]}"
+        data << "Content-length: #{content.length}"
+      end
+
+      if @options[:headers]
+        @options[:headers].each do |name,value|
+            data << "#{name}: #{value}"
+        end
+      end
+
+      data << "\r\n"
+
+      send_data data.join("\r\n") << content
+    end
+
+    def receive_error e
+      @error_callback.call(e) if @error_callback
+    end
+
+    def parse_stream_line ln
+      ln.strip!
+      unless ln.empty?
+        if ln[0,1] == '{' || ln[ln.length-1,1] == '}'
+          @stream << ln
+          if @stream[0,1] == '{' && @stream[@stream.length-1,1] == '}'
+            @each_item_callback.call(@stream) if @each_item_callback
+            @stream = ''
+          end
+        end
+      end
+    end
+
+    def reset_timeouts
+      set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
+      @nf_last_reconnect = @af_last_reconnect = nil
+      @reconnect_retries = 0
+    end
+
+    #
+    # URL and request components
+    #
+
+    # :filters => %w(miama lebron jesus)
+    # :oauth => {
+    #   :consumer_key    => [key],
+    #   :consumer_secret => [token],
+    #   :access_key      => [access key],
+    #   :access_secret   => [access secret]
+    # }
+    def oauth_header
+      uri = uri_base + @options[:path].to_s
+
+      # The hash SimpleOAuth accepts is slightly different from that of
+      # ROAuth.  To preserve backward compatability, fix the cache here
+      # so that the arguments passed in don't need to change.
+      oauth = {
+        :consumer_key => @options[:oauth][:consumer_key],
+        :consumer_secret => @options[:oauth][:consumer_secret],
+        :token => @options[:oauth][:access_key],
+        :token_secret => @options[:oauth][:access_secret]
+      }
+
+      data = ['POST', 'PUT'].include?(@options[:method]) ? params : {}
+
+      SimpleOAuth::Header.new(@options[:method], uri, data, oauth)
+    end
+
+    # Scheme (https if ssl, http otherwise) and host part of URL
+    def uri_base
+      "http#{'s' if @options[:ssl]}://#{@options[:host]}"
+    end
+
+    # Normalized query hash of escaped string keys and escaped string values
+    # nil values are skipped
+    def params
+      flat = {}
+      @options[:params].merge( :track => @options[:filters] ).each do |param, val|
+        next if val.to_s.empty? || (val.respond_to?(:empty?) && val.empty?)
+        val = val.join(",") if val.respond_to?(:join)
+        flat[param.to_s] = val.to_s
+      end
+      flat
+    end
+
+    def query
+      params.map{|param, value| [escape(param), escape(value)].join("=")}.sort.join("&")
+    end
+
+    def escape str
+      URI.escape(str.to_s, /[^a-zA-Z0-9\-\.\_\~]/)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,53 @@
+require 'rubygems'
+
+gem 'rspec', '>= 2.5.0'
+require 'rspec'
+require 'rspec/mocks'
+
+require 'twitter/json_stream'
+
+def fixture_path(path)
+  File.join(File.dirname(__FILE__), '..', 'fixtures', path)
+end
+
+def read_fixture(path)
+  File.read(fixture_path(path))
+end
+
+def connect_stream(opts={}, &blk)
+  EM.run {
+    opts.merge!(:host => Host, :port => Port)
+    stop_in = opts.delete(:stop_in) || 0.5
+    unless opts[:start_server] == false
+      EM.start_server Host, Port, JSONServer
+    end
+    @stream = JSONStream.connect(opts)
+    blk.call if blk
+    EM.add_timer(stop_in){ EM.stop }
+  }
+end
+
+def http_response(status_code, status_text, headers, body)
+  res = "HTTP/1.1 #{status_code} #{status_text}\r\n"
+  headers = {
+    "Content-Type"=>"application/json",
+    "Transfer-Encoding"=>"chunked"
+  }.merge(headers)
+  headers.each do |key,value|
+    res << "#{key}: #{value}\r\n"
+  end
+  res << "\r\n"
+  if headers["Transfer-Encoding"] == "chunked" && body.kind_of?(Array)
+    body.each do |data|
+      res << http_chunk(data)
+    end
+  else
+    res << body
+  end
+  res
+end
+
+def http_chunk(data)
+  # See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
+  "#{data.length.to_s(16)}\r\n#{data}\r\n"
+end

data/spec/twitter/json_stream_spec.rb

@@ -0,0 +1,335 @@
+require 'spec_helper.rb'
+require 'twitter/json_stream'
+
+include Twitter
+
+Host = "127.0.0.1"
+Port = 9550
+
+class JSONServer < EM::Connection
+  attr_accessor :data
+  def receive_data data
+    $recieved_data = data
+    send_data $data_to_send
+    EventMachine.next_tick {
+      close_connection if $close_connection
+    }
+  end
+end
+
+
+
+describe JSONStream do
+
+  context "authentication" do
+    it "should connect with basic auth credentials" do
+      connect_stream :auth => "username:password", :ssl => false
+      $recieved_data.should include('Authorization: Basic')
+    end
+
+    it "should connect with oauth credentials" do
+      oauth = {
+        :consumer_key => '1234567890',
+        :consumer_secret => 'abcdefghijklmnopqrstuvwxyz',
+        :access_key => 'ohai',
+        :access_secret => 'ohno'
+      }
+      connect_stream :oauth => oauth, :ssl => false
+      $recieved_data.should include('Authorization: OAuth')
+    end
+  end
+
+  context "on create" do
+
+    it "should return stream" do
+      EM.should_receive(:connect).and_return('TEST INSTANCE')
+      stream = JSONStream.connect {}
+      stream.should == 'TEST INSTANCE'
+    end
+
+    it "should define default properties" do
+      EM.should_receive(:connect).with do |host, port, handler, opts|
+        host.should == 'stream.twitter.com'
+        port.should == 443
+        opts[:path].should == '/1/statuses/filter.json'
+        opts[:method].should == 'GET'
+      end
+      stream = JSONStream.connect {}
+    end
+
+    it "should connect to the proxy if provided" do
+      EM.should_receive(:connect).with do |host, port, handler, opts|
+        host.should == 'my-proxy'
+        port.should == 8080
+        opts[:host].should == 'stream.twitter.com'
+        opts[:port].should == 443
+        opts[:proxy].should == 'http://my-proxy:8080'
+      end
+      stream = JSONStream.connect(:proxy => "http://my-proxy:8080") {}
+    end
+
+    it "should not trigger SSL until connection is established" do
+      connection = stub('connection')
+      EM.should_receive(:connect).and_return(connection)
+      stream = JSONStream.connect(:ssl => true)
+      stream.should == connection
+    end
+
+  end
+
+  context "on valid stream" do
+    attr_reader :stream
+    before :each do
+      $body = File.readlines(fixture_path("twitter/tweets.txt"))
+      $body.each {|tweet| tweet.strip!; tweet << "\r" }
+      $data_to_send = http_response(200, "OK", {}, $body)
+      $recieved_data = ''
+      $close_connection = false
+    end
+
+    it "should add no params" do
+      connect_stream :ssl => false
+      $recieved_data.should include('/1/statuses/filter.json HTTP')
+    end
+
+    it "should add custom params" do
+      connect_stream :params => {:name => 'test'},  :ssl => false
+      $recieved_data.should include('?name=test')
+    end
+
+    it "should parse headers" do
+      connect_stream :ssl => false
+      stream.code.should == 200
+      stream.headers.keys.map{|k| k.downcase}.should include('content-type')
+    end
+
+    it "should parse headers even after connection close" do
+      connect_stream :ssl => false
+      stream.code.should == 200
+      stream.headers.keys.map{|k| k.downcase}.should include('content-type')
+    end
+
+    it "should extract records" do
+      connect_stream :user_agent => 'TEST_USER_AGENT',  :ssl => false
+      $recieved_data.upcase.should include('USER-AGENT: TEST_USER_AGENT')
+    end
+
+    it 'should allow custom headers' do
+      connect_stream :headers => { 'From' => 'twitter-stream' },  :ssl => false
+      $recieved_data.upcase.should include('FROM: TWITTER-STREAM')
+    end
+
+    it "should deliver each item" do
+      items = []
+      connect_stream :ssl => false do
+        stream.each_item do |item|
+          items << item
+        end
+      end
+      # Extract only the tweets from the fixture
+      tweets = $body.map{|l| l.strip }.select{|l| l =~ /^\{/ }
+      items.size.should == tweets.size
+      tweets.each_with_index do |tweet,i|
+        items[i].should == tweet
+      end
+    end
+
+    it "should swallow StandardError exceptions when delivering items" do
+      expect do
+        connect_stream :ssl => false do
+          stream.each_item { |item| raise StandardError, 'error message' }
+        end
+      end.to_not raise_error
+    end
+
+
+    it "propagates out runtime errors when delivering items" do
+      expect do
+        connect_stream :ssl => false do
+          stream.each_item { |item| raise Exception, 'error message' }
+        end
+      end.to raise_error(Exception, 'error message')
+    end
+
+    it "should send correct user agent" do
+      connect_stream
+    end
+  end
+
+  shared_examples_for "network failure" do
+    it "should reconnect on network failure" do
+      connect_stream do
+        stream.should_receive(:reconnect)
+      end
+    end
+
+    it "should not reconnect on network failure when not configured to auto reconnect" do
+      connect_stream(:auto_reconnect => false) do
+        stream.should_receive(:reconnect).never
+      end
+    end
+
+    it "should reconnect with 0.25 at base" do
+      connect_stream do
+        stream.should_receive(:reconnect_after).with(0.25)
+      end
+    end
+
+    it "should reconnect with linear timeout" do
+      connect_stream do
+        stream.nf_last_reconnect = 1
+        stream.should_receive(:reconnect_after).with(1.25)
+      end
+    end
+
+    it "should stop reconnecting after 100 times" do
+      connect_stream do
+        stream.reconnect_retries = 100
+        stream.should_not_receive(:reconnect_after)
+      end
+    end
+
+    it "should notify after reconnect limit is reached" do
+      timeout, retries = nil, nil
+      connect_stream do
+        stream.on_max_reconnects do |t, r|
+          timeout, retries = t, r
+        end
+        stream.reconnect_retries = 100
+      end
+      timeout.should == 0.25
+      retries.should == 101
+    end
+  end
+
+  context "on network failure" do
+    attr_reader :stream
+    before :each do
+      $data_to_send = ''
+      $close_connection = true
+    end
+
+    it "should timeout on inactivity" do
+      connect_stream :stop_in => 1.5 do
+        stream.should_receive(:reconnect)
+      end
+    end
+
+    it "should not reconnect on inactivity when not configured to auto reconnect" do
+      connect_stream(:stop_in => 1.5, :auto_reconnect => false) do
+        stream.should_receive(:reconnect).never
+      end
+    end
+
+    it "should reconnect with SSL if enabled" do
+      connect_stream :ssl => true do
+        stream.should_receive(:start_tls).twice
+      end
+    end
+
+    it_should_behave_like "network failure"
+  end
+
+  context "on no data received" do
+    attr_reader :stream
+    before :each do
+      $data_to_send = ''
+      $close_connection = false
+    end
+
+    it "should call no data callback after no data received for 90 seconds" do
+      connect_stream :stop_in => 6 do
+        stream.last_data_received_at = Time.now - 88
+        stream.should_receive(:no_data).once
+      end
+    end
+
+  end
+
+  context "on server unavailable" do
+
+    attr_reader :stream
+
+    # This is to make it so the network failure specs which call connect_stream
+    # can be reused. This way calls to connect_stream won't actually create a
+    # server to listen in.
+    def connect_stream_without_server(opts={},&block)
+      connect_stream_default(opts.merge(:start_server=>false),&block)
+    end
+    alias_method :connect_stream_default, :connect_stream
+    alias_method :connect_stream, :connect_stream_without_server
+
+    it_should_behave_like "network failure"
+  end
+
+  context "on application failure" do
+    attr_reader :stream
+    before :each do
+      $data_to_send = "HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Basic realm=\"Firehose\"\r\n\r\n"
+      $close_connection = false
+    end
+
+    it "should reconnect on application failure 10 at base" do
+      connect_stream :ssl => false do
+        stream.should_receive(:reconnect_after).with(10)
+      end
+    end
+
+    it "should not reconnect on application failure 10 at base when not configured to auto reconnect" do
+      connect_stream  :ssl => false, :auto_reconnect => false do
+        stream.should_receive(:reconnect_after).never
+      end
+    end
+
+    it "should reconnect with exponential timeout" do
+      connect_stream :ssl => false do
+        stream.af_last_reconnect = 160
+        stream.should_receive(:reconnect_after).with(320)
+      end
+    end
+
+    it "should not try to reconnect after limit is reached" do
+      connect_stream :ssl => false do
+        stream.af_last_reconnect = 320
+        stream.should_not_receive(:reconnect_after)
+      end
+    end
+  end
+
+  context "on stream with chunked transfer encoding" do
+    attr_reader :stream
+    before :each do
+      $recieved_data = ''
+      $close_connection = false
+    end
+
+    it "should ignore empty lines" do
+      body_chunks = ["{\"screen"+"_name\"",":\"user1\"}\r\r\r{","\"id\":9876}\r\r"]
+      $data_to_send = http_response(200,"OK",{},body_chunks)
+      items = []
+      connect_stream :ssl => false do
+        stream.each_item do |item|
+          items << item
+        end
+      end
+      items.size.should == 2
+      items[0].should == '{"screen_name":"user1"}'
+      items[1].should == '{"id":9876}'
+    end
+
+    it "should parse full entities even if split" do
+      body_chunks = ["{\"id\"",":1234}\r{","\"id\":9876}"]
+      $data_to_send = http_response(200,"OK",{},body_chunks)
+      items = []
+      connect_stream :ssl => false do
+        stream.each_item do |item|
+          items << item
+        end
+      end
+      items.size.should == 2
+      items[0].should == '{"id":1234}'
+      items[1].should == '{"id":9876}'
+    end
+  end
+
+end

data/twitter-stream.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{cantino-twitter-stream}
+  s.version = "0.1.15"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Andrew Cantino", "Vladimir Kolesnikov"]
+  s.date = %q{2012-04-10}
+  s.description = %q{Simple Ruby client library for twitter streaming API. Uses EventMachine for connection handling. Adheres to twitter's reconnection guidline. JSON format only.}
+  s.summary = %q{Twitter realtime API client- updated to use nerver version of http_parser}
+  s.homepage = %q{http://github.com/cantino/twitter-stream}
+  s.email = %q{voloko@gmail.com}
+
+  s.platform                  = Gem::Platform::RUBY
+  s.rubygems_version          = %q{1.3.6}
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.3.6") if s.respond_to? :required_rubygems_version=
+
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.extra_rdoc_files = ["README.markdown", "LICENSE"]
+
+  s.add_runtime_dependency('eventmachine', ">= 0.12.8")
+  s.add_runtime_dependency('simple_oauth', '~> 0.2.0')
+  s.add_runtime_dependency('http_parser.rb', '~> 0.6.0')
+  s.add_development_dependency('rspec', "~> 2.5.0")
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end
+

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: cantino-twitter-stream
+version: !ruby/object:Gem::Version
+  version: 0.1.15
+platform: ruby
+authors:
+- Andrew Cantino
+- Vladimir Kolesnikov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.12.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.12.8
+- !ruby/object:Gem::Dependency
+  name: simple_oauth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: http_parser.rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+description: Simple Ruby client library for twitter streaming API. Uses EventMachine
+  for connection handling. Adheres to twitter's reconnection guidline. JSON format
+  only.
+email: voloko@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.markdown
+- LICENSE
+files:
+- .gemtest
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE
+- README.markdown
+- Rakefile
+- VERSION
+- examples/reader.rb
+- fixtures/twitter/tweets.txt
+- lib/twitter/json_stream.rb
+- spec/spec_helper.rb
+- spec/twitter/json_stream_spec.rb
+- twitter-stream.gemspec
+homepage: http://github.com/cantino/twitter-stream
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Twitter realtime API client- updated to use nerver version of http_parser
+test_files:
+- spec/spec_helper.rb
+- spec/twitter/json_stream_spec.rb