birdgrinder 0.1.1.1 → 0.1.2

data/lib/bird_grinder/client.rb

@@ -38,6 +38,7 @@ module BirdGrinder
     # Forwards a given message type (with options) to each handler,
     # storing the current id if changed.
     def receive_message(type, options = BirdGrinder::Nash.new)
+      options = options.to_nash if options.respond_to?(:to_nash)
       logger.debug "receiving message: #{type.inspect} - #{options.id? ? options.id : 'unknown id'}"
       dispatch(type.to_sym, options)
       update_stored_id_for(type, options.id) if options.id?
@@ -59,6 +60,13 @@ module BirdGrinder
       @tweeter.search(q, opts)
     end
     
+    # Returns the streaming api instance
+    #
+    # @see BirdGrinder::Tweeter#streaming
+    def streaming
+      @tweeter.streaming
+    end
+    
     # Tweets some text as the current user
     #
     # @see BirdGrinder::Tweeter#tweet

data/lib/bird_grinder/tweeter/stream_processor.rb

@@ -3,9 +3,10 @@ module BirdGrinder
     class StreamProcessor
       is :loggable
       
-      def initialize(parent, stream_name)
+      def initialize(parent, stream_name, stream_meta = {})
         @parent = parent
         @stream_name = stream_name.to_sym
+        @stream_meta = stream_meta.to_nash
         setup_parser
       end
       
@@ -27,6 +28,7 @@ module BirdGrinder
         end
         processed.stream_type = stream_type
         processed.streaming_source = @stream_name
+        processed.meta = @stream_meta
         @parent.delegate.receive_message(:incoming_stream, processed)
       end
    

data/lib/bird_grinder/tweeter/streaming.rb

@@ -1,4 +1,4 @@
-require 'bird_grinder/tweeter/stream_processor'
+require 'bird_grinder/tweeter/streaming_request'
 
 module BirdGrinder
   class Tweeter
@@ -24,14 +24,14 @@ module BirdGrinder
       #
       # @param [Hash] opts extra options for the query
       def sample(opts = {})
-        get(:sample, opts)
+        stream(:sample, opts)
       end
       
       # Start processing the filter stream
       #
       # @param [Hash] opts extra options for the query
       def filter(opts = {})
-        get(:filter, opts)
+        stream(:filter, opts)
       end
       
       # Start processing the filter stream with a given follow
@@ -42,7 +42,7 @@ module BirdGrinder
         opts = args.extract_options!
         opts[:follow] = args.join(",")
         opts[:path] = :filter
-        get(:follow, opts)
+        stream(:follow, opts)
       end
       
       # Starts tracking a specific query.
@@ -51,22 +51,20 @@ module BirdGrinder
       def track(query, opts = {})
         opts[:track] = query
         opts[:path] = :filter
-        get(:track, opts)
+        stream(:track, opts)
       end
       
       protected
       
+      def stream(name, opts = {})
+        req = StreamingRequest.new(@parent, name, opts)
+        yield req if block_given?
+        req.perform
+        req
+      end
+      
       def get(name, opts = {}, attempts = 0)
-        logger.debug "Getting stream #{name} w/ options: #{opts.inspect}"
-        path = opts.delete(:path)
-        processor = StreamProcessor.new(@parent, name)
-        http_opts = {
-          :head => {'Authorization' => @parent.auth_credentials}
-        }
-        http_opts[:query] = opts if opts.present?
-        url = streaming_base_url / api_version.to_s / "statuses" / "#{path || name}.json"
-        http = EventMachine::HttpRequest.new(url).get(http_opts)
-        http.stream(&processor.method(:receive_chunk))
+        
       end
       
     end

data/lib/bird_grinder/tweeter/streaming_request.rb

@@ -0,0 +1,109 @@
+require 'bird_grinder/tweeter/stream_processor'
+require 'cgi'
+
+module BirdGrinder
+  class Tweeter
+    class StreamingRequest
+      is :loggable
+      
+      INITIAL_DELAYS = {:http => 10,  :network => 0.25}
+      MAX_DELAYS     = {:http => 240, :network => 16}
+      DELAY_CALCULATOR = {
+        :http    => L { |v| v * 2 },
+        :network => L { |v| v + INITIAL_DELAYS[:network] }
+      }
+      
+      def initialize(parent, name, options = {})
+        logger.debug "Creating stream '#{name}' with options: #{options.inspect}"
+        @parent         = parent
+        @name           = name
+        @path           = options.delete(:path) || :name
+        @metadata       = options.delete(:metadata) || {}
+        @options        = options
+        @failure_delay  = nil
+        @failure_count  = 0
+        @failure_reason = nil
+      end
+      
+      def perform
+        logger.debug "Preparing to start stream"
+        @stream_processor = nil
+        type = request_method
+        http = create_request.send(type, http_options(type))
+        # Handle failures correctly so we can back off
+        @current_request = http
+        http.errback  { fail!(:network)}
+        http.callback { http.response_header.status > 299 ? fail!(:http) : perform }
+        http.stream { |c| receive_chunk(c) }
+      end
+      
+      def fail!(type)
+        logger.debug "Streaming failed with #{type}"
+        if @failure_count == 0 || @failure_reason != type
+          logger.debug "Instantly restarting (#{@failure_count == 0  ? "First failure" : "Different type"})"
+          EM.next_tick { perform }
+        else
+          @failure_delay ||= INITIAL_DELAYS[type]
+          logger.debug "Restarting stream in #{@failure_delay} seconds"
+          logger.debug "Adding timer to restart in #{@failure_delay} seconds"
+          EM.add_timer(@failure_delay) { perform }
+          potential_new_delay = DELAY_CALCULATOR[type].call(@failure_delay)
+          @failure_delay = [potential_new_delay, MAX_DELAYS[type]].min
+          logger.debug "Next delay is #{@failure_delay}"
+        end
+        @failure_count += 1
+        @failure_reason = type
+        logger.debug "Failed #{@failure_count} times with #{@failure_reason}"
+      end
+      
+      def create_request
+        EventMachine::HttpRequest.new(full_url)
+      end
+      
+      def stream_processor
+        @stream_processor ||= StreamProcessor.new(@parent, @name, @metadata)
+      end
+      
+      def receive_chunk(c)
+        return unless @current_request.response_header.status == 200
+        if !@failure_reason.nil?
+          @failure_reason = nil
+          @failure_delay  = nil
+          @failure_count  = 0
+        end
+        stream_processor.receive_chunk(c)
+      end
+      
+      def default_request_options
+        {:head => {'Authorization' => @parent.auth_credentials}}
+      end
+      
+      def http_options(type)
+        base = self.default_request_options
+        if @options.present?
+          if type == :get
+            base[:query] = @options
+          else
+            base[:head].merge! 'Content-Type'  => "application/x-www-form-urlencoded"
+            base[:body] = body = {}
+            @options.each_pair { |k,v| body[CGI.escape(k.to_s)] = CGI.escape(v) }
+          end
+        end
+        base
+      end
+      
+      def request_method
+        {:filter   => :post,
+         :sample   => :get,
+         :firehose => :get,
+         :retweet  => :get
+        }.fetch(@path, :get)
+      end
+      
+      def full_url
+        @full_url ||= (Streaming.streaming_base_url / Streaming.api_version.to_s / "statuses" / "#{@path}.json")
+      end
+      
+    end
+  end
+end

data/lib/bird_grinder/tweeter.rb

@@ -1,4 +1,5 @@
 require 'uri'
+require 'cgi'
 
 module BirdGrinder
   # An asynchronous, delegate-based twitter client that uses
@@ -57,7 +58,7 @@ module BirdGrinder
       user = user.to_s.strip
       logger.info "Following '#{user}'"
       post("friendships/create.json", opts.merge(:screen_name => user)) do
-        delegate.receive_message(:outgoing_follow, :user => user)
+        delegate.receive_message(:outgoing_follow, {:user => user}.to_nash)
       end
     end
     
@@ -69,7 +70,7 @@ module BirdGrinder
       user = user.to_s.strip
       logger.info "Unfollowing '#{user}'"
       post("friendships/destroy.json", opts.merge(:screen_name => user)) do
-        delegate.receive_message(:outgoing_unfollow, :user => user)
+        delegate.receive_message(:outgoing_unfollow, {:user => user}.to_nash)
       end
     end
     
@@ -95,7 +96,7 @@ module BirdGrinder
       user = user.to_s.strip
       logger.debug "DM'ing #{user}: #{text}"
       post("direct_messages/new.json", opts.merge(:user => user, :text => text)) do
-        delegate.receive_message(:outgoing_direct_message, :user => user, :text => text)
+        delegate.receive_message(:outgoing_direct_message, {:user => user, :text => text}.to_nash)
       end
     end
     
@@ -223,7 +224,7 @@ module BirdGrinder
     
     def post(path, params = {}, &blk)
       real_params = {}
-      params.each_pair { |k,v| real_params[URI.encode(k.to_s)] = URI.encode(v) }
+      params.each_pair { |k,v| real_params[CGI.escape(k.to_s)] = CGI.escape(v) }
       http = request(path).post({
         :head => {
           'Authorization' => @auth_credentials,

data/lib/bird_grinder.rb

@@ -8,7 +8,7 @@ require 'em-http'
 module BirdGrinder
   include Perennial
   
-  VERSION = [0, 1, 1, 1]
+  VERSION = [0, 1, 2]
 
   def self.version(include_minor = false)
     VERSION[0, (include_minor ? 4 : 3)].join(".")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: birdgrinder
 version: !ruby/object:Gem::Version 
-  version: 0.1.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - Darcy Laycock
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-19 00:00:00 +08:00
+date: 2009-10-24 00:00:00 +08:00
 default_executable: birdgrinder
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -94,6 +94,7 @@ files:
 - lib/bird_grinder/tweeter/search.rb
 - lib/bird_grinder/tweeter/stream_processor.rb
 - lib/bird_grinder/tweeter/streaming.rb
+- lib/bird_grinder/tweeter/streaming_request.rb
 - lib/bird_grinder/tweeter.rb
 - lib/bird_grinder.rb
 - lib/birdgrinder.rb