typho-twitter 0.2.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,24 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC
+
+test/oauth.yaml
+.rvmrc

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 shock
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,61 @@
+= Typho Twitter
+
+== What
+
+This is a RubyGem to simplify sending parallel batches of requests to the Twitter API in Ruby applications.
+
+It is based on Typhoeus and OAuth.
+
+It is currently a work in progress.  Comments, suggestions, and feedback are welcome and encouraged.
+
+== Why
+
+Some applications need to send lots of individual requests to the Twitter API to do things such as retrieve details from a group of users, or get the recent statuses for a group of users.
+
+For a sizable number of requests, doing this serially is extremely slow.  TyphoTwitter lets you perform a batch of like requests in parallel, drastically reducing the amount of time it takes to perform the same number of requests.
+
+== Installing
+
+  sudo gem install typho-twitter
+
+The source code is hosted on GitHub: http://github.com/capitalthought/typho-twitter
+
+== The basics
+
+
+== Demonstration of usage
+
+Create a TyphoTwitter instance.  If you need to authorize:
+
+  @typho_twitter = TyphoTwitter.new( 
+    :oauth=>{
+      :consumer_key=>'XXXXXXX', 
+      :consumer_secret=>'YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY', 
+      :token=>'XXXXXXX-YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY', 
+      :secret=>'ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ',
+      :site=>'http://example.com'
+    } 
+  )
+  screen_name_array = %w[02Blazer 080808news 0Amna0 100PercentTX 1043LaQueBuena 1049TheHorn 1070thefan 1070WINA 10jackrussel 10rWfe 10tonreverb 1337studios 141chars 1450whtc 1660THEFAN 16mthsapart 1968mike 1capplegate 1LUVMRWAY 1MattHopkins 1OneStone 1realestateteam 1stbassguitar 1stBrand 1ststepsmoney 1TeeTime 1weightliftin 2001MUgrad 203klender 20thCFlicks]
+  responses = @typho_twitter.get_users_show( screen_name_array )
+  responses.each do |response|
+    puts response.to_s
+  end
+
+== More Information
+
+* RDoc: http://rdoc.info/projects/capitalthought/typho-twitter/
+
+== How to submit patches
+
+The source code is hosted on the GitHub: http://github.com/capitalthought/typho-twitter
+
+To submit a patch, please fork the typho-twitter project and create a patch with tests. Once you're happy with it send a pull request and post a message to the google group.
+
+== License
+
+This code is free to use under the terms of the MIT license. 
+
+== Contact
+
+Comments are welcome. Send an email to me at typho-twitter@wdd.oib.com

data/Rakefile

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "typho-twitter"
+    gem.summary = %Q{Parallel twitter client using Typhoeus.}
+    gem.description = %Q{A Twitter client for performing a batch of Twitter calls in parallel.}
+    gem.email = "billdoughty@capitalthought.com"
+    gem.homepage = "http://github.com/capitalthought/typho-twitter"
+    gem.authors = ["shock"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_dependency "wdd-ruby-ext", ">= 0.3.1"
+    gem.add_dependency "oauth", ">=0.4.3"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "typho-twitter #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.1

data/lib/typho-twitter.rb

@@ -0,0 +1 @@
+require 'typho-twitter/typho-twitter'

data/lib/typho-twitter/typho-twitter.rb

@@ -0,0 +1,474 @@
+# Note the tests at the bottom.  You can test this class by running it standalone in the interpreter.
+
+require 'rubygems'
+require 'wdd-ruby-ext'
+require "cgi"
+require "net/http"
+require "uri"
+require "time"
+require "pp"
+require 'json'
+require 'base64'
+require 'logger'
+require "thread"
+require "oauth"
+require "oauth/request_proxy/typhoeus_request"
+
+# Class to abstract access to Twitter's Web Traffic API.
+# Makes use of the Typhoeus gem to enable concurrent API calls.
+
+class TyphoTwitter
+  
+  include WDD::Utils
+  
+  private
+  
+  def logger  
+    @logger
+  end
+  
+  def puts message
+    logger.debug message
+  end
+  
+  public
+  
+  class HTTPException < RuntimeError
+    attr :code
+    attr :body
+    
+    def initialize( code, body )
+      @code = code
+      @body = body
+      super( "#{code} - #{body}" )
+    end
+  end
+  
+  class TwitterException < RuntimeError
+    attr :code
+    attr :body
+    
+    def initialize( code, body )
+      @code = code
+      @body = body
+      super( "#{code} - #{body}" )
+    end
+  end
+    
+  attr :login
+  attr :password
+  attr :headers
+  
+  # Constants 
+  DEFAULT_REQUEST_TIMEOUT = 20000
+  DEFAULT_CONCURRENCY_LIMIT = 40
+  MAX_RETRIES = 10
+  
+  # Create a TyphoTwitter instance.
+  # +options+ :
+  #   :request_timeout Request timeout value in miliseconds
+  #   :request_timeout Request timeout value in miliseconds
+  #   :concurrency_limit Maximum number of concurrent Typhoeus requests
+  #   :logger Logger to use - defaults to standard out with DEBUG level if not specified.
+  #   :oauth Used to authorize with Twitter API.
+  #     :consumer_key Consumer key for your application
+  #     :consumer_secret Consumer secret for your appilcation
+  #     :token Access token for the Twitter user to authenticate with
+  #     :secret Access token secret
+  #     :site The site URL for your application
+  def initialize options={}
+    if options[:oauth]
+      @oauth_options = {}
+      @oauth_options[:consumer] = OAuth::Consumer.new( options[:oauth][:consumer_key], options[:oauth][:consumer_secret], :site => options[:oauth][:site] )
+      @oauth_options[:access_token] = OAuth::AccessToken.from_hash( @oauth_options[:consumer], :oauth_token=>options[:oauth][:token], :oauth_token_secret=>options[:oauth][:secret] )
+    end
+    @headers = {}
+    @options = options
+    @request_timeout = options[:request_timeout] || DEFAULT_REQUEST_TIMEOUT
+    @concurrency_limit = options[:concurrency_limit] || DEFAULT_CONCURRENCY_LIMIT
+    @logger = options[:logger]
+    if @logger.nil?
+      $stdout.sync = true
+      @logger = Logger.new( $stdout )
+      @logger.level = Logger::DEBUG
+    end
+  end
+
+  # Executes a batch of Twitter calls.  Automatically handles timeout_retries when possible on failures.
+  # Returns a hash where each +data_array+ element is a key mapping to either 
+  # a hash containing the results of the twitter call or a TwitterException object of the twitter call that failed.
+  # If +data_array+ is bigger than the concurrency_limit set in the TyphoTwitter constructor, it is broken
+  # up into batches of requests by Typhoeus automatically.
+  # +data_array+ - An array of data inputs, one for each twitter call
+  # +&block+ - A block that accepts an element of +data_array+ and returns a Tyhpoeus::Request object.
+  def typho_twitter_batch data_array, &block
+    
+    json_results = {}
+    timeout_retries = 0
+    time_gate = WDD::Utils::TimeGate.new
+    rate_limit_exceeded = false
+    while data_array.length > 0
+      puts "Waiting on rate limiting Time Gate"
+      time_gate.wait
+      puts "Time Gate released"
+      hydra = Typhoeus::Hydra.new(:max_concurrency => @concurrency_limit)
+      hydra.disable_memoization
+      
+      retries = 0
+      
+      timed_out_inputs = Queue.new
+      data_array.each do |data_input|
+        request = yield( data_input )
+        if @oauth_options
+          oauth_params = {:consumer => @oauth_options[:consumer], :token => @oauth_options[:access_token]}
+          oauth_helper = OAuth::Client::Helper.new(request, oauth_params.merge(:request_uri => request.url))
+          request.headers.merge!({"Authorization" => oauth_helper.header}) # Signs the request
+        end
+        # printvar :request, request
+        request.on_complete do |response|
+          puts "[#{response.code}] - #{request.url}"
+          case response.code
+          when 200:
+            begin
+              json_object = JSON.parse( response.body )
+              json_results[data_input] = json_object
+              retries = 0
+            rescue JSON::ParserError
+              if timeout_retries < MAX_RETRIES
+                timed_out_inputs.push data_input
+              else
+                json_results[data_input] = $!
+                logger.error "#{$!.inspect}"
+                logger.error "#{$!.backtrace.join("\n")}"
+                logger.error "response.body: •#{response.body}•"
+              end
+            end
+          when 0:
+            logger.debug "**** Twitter Timeout (#{response.code}) for #{data_input}."
+            logger.debug "Response body: #{response.body}"
+            if timeout_retries < MAX_RETRIES
+              timed_out_inputs.push data_input
+            else
+              json_results[data_input] = TwitterException.new(response.code, response.body)
+            end
+          when 400:
+            logger.debug "**** Twitter Rate Limit Exceeded (#{response.code}) for #{data_input}."
+            rate_limit_exceeded = true
+            json_results[data_input] = TwitterException.new(response.code, response.body)
+          when 401:
+            logger.debug "**** Twitter Authorization Failed (#{response.code}) for #{data_input}."
+            logger.debug "Request URL: #{request.url}"
+            json_results[data_input] = TwitterException.new(response.code, response.body)
+          when 404:
+            logger.debug "Unknown data_input: #{data_input}"              
+            logger.debug "Request URL: #{request.url}"
+            json_results[data_input] = TwitterException.new(response.code, response.body)
+          when 502:
+            logger.debug "Twitter Over capacity (#{response.code}) for data_input: #{data_input}.  Will retry."
+            logger.debug "Request URL: #{request.url}"
+            retries += 1
+            if retries < MAX_RETRIES
+              sleep_time = retries**2
+              logger.debug "Will retry after #{sleep_time} seconds."
+              sleep sleep_time
+              hydra.queue request
+            else
+              json_results[data_input] = TwitterException.new(response.code, response.body)
+            end
+          when 503:
+            logger.debug "Twitter Service Unavailable (#{response.code}) for data_input: #{data_input}.  Will retry."
+            logger.debug "Request URL: #{request.url}"
+            retries += 1
+            if retries < MAX_RETRIES
+              sleep_time = retries**2
+              logger.debug "Will retry after #{sleep_time} seconds."
+              sleep sleep_time
+              hydra.queue request
+            else
+              json_results[data_input] = TwitterException.new(response.code, response.body)
+            end
+          when 500:
+            logger.debug "Twitter server error for data_input: #{data_input}.  Will retry."
+            logger.debug "Request URL: #{request.url}"
+            retries += 1
+            if retries < MAX_RETRIES
+              sleep_time = retries**2
+              logger.debug "Will retry after #{sleep_time} seconds."
+              sleep sleep_time
+              hydra.queue request
+            else
+              json_results[data_input] = TwitterException.new(response.code, response.body)
+            end
+          else
+            logger.error "Unexpected HTTP result code: #{response.code}\n#{response.body}"
+            logger.error "Request URL: #{request.url}"
+            json_results[data_input] = TwitterException.new(response.code, response.body)
+          end        
+        end
+        hydra.queue request
+      end
+      logger.debug "+++ Running Hydra."
+      hydra.run
+      logger.debug "--- Hydra run complete."
+      data_array = []
+      if timed_out_inputs.size > 0
+        logger.debug "#{timed_out_inputs.size} ERRORS encountered."
+        while !timed_out_inputs.empty?
+          failed_input = timed_out_inputs.pop
+          logger.debug "Reloading #{failed_input}"
+          data_array << failed_input
+        end
+        timeout_retries += 1
+        sleep_time = timeout_retries ** 2
+        logger.debug "Will retry after #{sleep_time} seconds."
+        sleep sleep_time
+      end
+    end
+    json_results
+  end  
+  
+
+  # Retrieves user profile data for a group of Twitter users.
+  # +twitter_id_array+ = An array twitter user ids, one for each user to get data for.  Can be user_ids or screen_names.
+  # Returns a results Hash (see typho_twitter_batch)
+  def get_users_show twitter_id_array
+    typho_twitter_batch( twitter_id_array ) do |twitter_id|
+      if twitter_id.is_a? Fixnum
+        request = Typhoeus::Request.new("http://twitter.com/users/show.json?user_id=#{twitter_id}",
+          :headers => @headers,
+          :timeout => @request_timeout # miliseconds          
+        )
+      else
+        request = Typhoeus::Request.new("http://twitter.com/users/show.json?screen_name=#{twitter_id}",
+          :timeout => @request_timeout, # miliseconds          
+          :headers => @headers
+        )
+      end
+      request
+    end
+  end  
+
+  # Retrieves the followers records for a group of Twitter users.
+  # +twitter_id_array+ = An array twitter user ids, one for each user to get data for
+  def get_statuses_followers twitter_id_array, limit=nil
+    master_results = {}
+    process_statuses_followers( twitter_id_array ) do |twitter_id, results|
+      master_results[twitter_id] ||= []
+      if results.is_a? TwitterException
+        master_results[twitter_id] = results
+        false
+      else
+        master_results[twitter_id] += results
+        if limit && master_results[twitter_id].length >= limit
+          master_results[twitter_id] = master_results[twitter_id].slice(0, limit)
+          continue = false
+        else
+          continue = true
+        end
+        logger.debug "#{twitter_id} - #{master_results[twitter_id].length} followers retrieved."
+        continue
+      end
+    end
+    master_results
+  end  
+
+  # Retrieves the followers records for a group of twitter_ids from Twitter and feeds them to the supplied
+  # block one page at a time.  The block passed is expected to return a true or false value.  If it 
+  # returns true, fetching of followers will continue for that twitter_id.  If it returns false, fetching
+  # of followers will be aborted for that twitter_id only.  This allows a batch of fetches to be started for
+  # multiple users.  Fetching of individual user's followers may be aborted while continuing the others.
+  # +twitter_ids+ = An array twitter twitter_ids, one for each user to get data for.
+  #
+  # Returns nil.
+  #
+  # eg.
+  #
+  #   process_statuses_followers( ['bdoughty', 'joshuabaer'] ) do |twitter_id, followers|
+  #     puts "Twitter user #{twitter_id}"
+  #     continue = true
+  #     followers.each do |follower|
+  #       continue = false if follower[:twitter_id] == 'needle'
+  #     end
+  #     continue 
+  #   end
+  def process_statuses_followers twitter_id_array, &block
+
+    raise "You must supply a block to this method." if !block_given?    
+    # Track the proper Twitter API cursor for each twitter_id.  Twitter requests an initial cursor of -1 (to begin paging)
+    cursor_tracker = {}
+    twitter_id_array.each do |twitter_id|
+      cursor_tracker[twitter_id] = -1
+    end
+    
+    while( cursor_tracker.size > 0 )
+      twitter_results = typho_twitter_batch( cursor_tracker.keys ) do |twitter_id|
+        if twitter_id.is_a? Fixnum
+          request = Typhoeus::Request.new("http://twitter.com/statuses/followers.json?cursor=#{cursor_tracker[twitter_id]}&user_id=#{twitter_id}",
+            :headers => @headers,
+            :timeout => @request_timeout
+          )
+        else
+          request = Typhoeus::Request.new("http://twitter.com/statuses/followers.json?cursor=#{cursor_tracker[twitter_id]}&screen_name=#{twitter_id}",
+            :timeout => @request_timeout,
+            :headers => @headers
+          )
+        end
+      end
+      cursor_tracker = {}
+      twitter_results.each do |twitter_id, results|
+        next_cursor = 0
+        if results.is_a?( Hash ) && results['users'] && results['users'].length > 0
+          next_cursor = results["next_cursor"]
+          continue = yield( twitter_id, results['users'] )
+        else
+          continue = yield( twitter_id, results ) # return the exception
+        end
+        if next_cursor != 0 && continue
+          cursor_tracker[twitter_id] = next_cursor
+        else
+          cursor_tracker.delete( twitter_id ) # remove the twitter_id from processing
+        end
+      end
+    end
+        
+    nil
+  end  
+
+  # Retrieves the followers ids for a group of twitter_ids from Twitter and feeds them to the supplied
+  # block one page at a time.  The block passed is expected to return a true or false value.  If it 
+  # returns true, fetching of follower ids will continue for that twitter_id.  If it returns false, fetching
+  # of follower ids will be aborted for that twitter_id only.  This allows a batch of fetches to be started for
+  # multiple users.  Fetching of individual user's followers ids may be aborted while continuing the others.
+  # +twitter_ids+ = An array twitter twitter_ids, one for each user to get data for.
+  #
+  # Returns nil.
+  #
+  # eg.
+  #
+  #   process_followers_ids( ['bdoughty', 'joshuabaer'] ) do |twitter_id, follower_ids|
+  #     puts "Twitter user #{twitter_id}"
+  #     continue = true
+  #     follower_ids.each do |follower_id|
+  #       continue = false if follower_id == SOME_TWITTER_USER_ID
+  #     end
+  #     continue 
+  #   end
+  def process_followers_ids twitter_id_array, &block
+
+    raise "You must supply a block to this method." if !block_given?    
+    # Track the proper Twitter API cursor for each twitter_id.  Twitter requests an initial cursor of -1 (to begin paging)
+    cursor_tracker = {}
+    twitter_id_array.each do |twitter_id|
+      cursor_tracker[twitter_id] = -1
+    end
+    
+    while( cursor_tracker.size > 0 )
+      twitter_results = typho_twitter_batch( cursor_tracker.keys ) do |twitter_id|
+        if twitter_id.is_a? Fixnum
+          request = Typhoeus::Request.new("http://twitter.com/followers/ids.json?cursor=#{cursor_tracker[twitter_id]}&user_id=#{twitter_id}",
+            :headers => @headers,
+            :timeout => @request_timeout
+          )
+        else
+          request = Typhoeus::Request.new("http://twitter.com/followers/ids.json?cursor=#{cursor_tracker[twitter_id]}&screen_name=#{twitter_id}",
+            :headers => @headers,
+            :timeout => @request_timeout
+          )
+        end
+      end
+      cursor_tracker = {}
+      twitter_results.each do |twitter_id, results|
+        next_cursor = 0
+        if results.is_a?( Hash ) && results['ids'] && results['ids'].length > 0
+          next_cursor = results["next_cursor"]
+          continue = yield( twitter_id, results['ids'] )
+        else
+          continue = yield( twitter_id, results ) # return the exception
+        end
+        if next_cursor != 0 && continue
+          cursor_tracker[twitter_id] = next_cursor
+        else
+          cursor_tracker.delete( twitter_id ) # remove the twitter_id from processing
+        end
+      end
+    end
+        
+    nil
+  end  
+
+  # Retrieves all timeline updates for a group of twitter_ids from Twitter.
+  # This method calls process_statuses_user_timeline() with a block to aggregate the updates.
+  # +twitter_id_array+ = An array twitter user ids, one for each user to get data for
+  # Returns aggregated updates as a Hash with twitter_ids as keys, and arrays of updates as values.
+  # If an unresolvable exception occurred fetching a particular twitter_id, then the resulting TwitterException
+  # is returned for that screen name instead of an array of updates.
+  def get_statuses_user_timeline twitter_id_array
+    master_results = {}
+    process_statuses_user_timeline( twitter_id_array ) do |twitter_id, results|
+      master_results[twitter_id] ||= []
+      if results.is_a? TwitterException
+        master_results[twitter_id] = results
+        false
+      else
+        master_results[twitter_id] += results
+        true
+      end
+    end
+    master_results
+  end
+  
+  # Retrieves the timeline updates for a group of twitter_ids from Twitter and feeds them to the supplied
+  # block one page at a time.  The block passed is expected to return a true or false value.  If it 
+  # returns true, fetching of updates will continue for that twitter_id.  If it returns false, fetching
+  # of updates will be aborted for that twitter_id only.  This allows a batch of fetches to be started for
+  # multiple users.  Fetching of individual user's updates may be aborted while continuing the others.
+  # +twitter_ids+ = An array twitter user ids, one for each user to get data for.
+  #
+  # Returns nil.
+  #
+  # eg.
+  #
+  #   process_statuses_user_timeline( ['bdoughty', 'joshuabaer'] ) do |twitter_id, updates|
+  #     puts "Twitter user #{twitter_id}"
+  #     updates.each do |update|
+  #       # do something with each status update
+  #     end
+  #     (twitter_id == 'bdoughty')  # block return value - aborts 'joshuabaer' after the first page, continues 'bdoughty'
+  #   end
+  def process_statuses_user_timeline twitter_ids, &block
+    page = 0
+    count = 200
+    while twitter_ids.length > 0
+      page += 1 # Twitter starts with page 1
+      logger.debug "Getting page #{page} for timelines."
+      twitter_results = typho_twitter_batch( twitter_ids ) do |twitter_id|
+        if twitter_id.is_a? Fixnum
+          request = Typhoeus::Request.new("http://twitter.com/statuses/user_timeline.json?user_id=#{twitter_id}&page=#{page}&count=#{count}",
+            :headers => @headers,
+            :timeout => @request_timeout
+          )
+        else
+          request = Typhoeus::Request.new("http://twitter.com/statuses/user_timeline.json?screen_name=#{twitter_id}&page=#{page}&count=#{count}",
+            :headers => @headers,
+            :timeout => @request_timeout
+          )
+        end
+      end
+
+      twitter_ids = []
+      twitter_results.each do |twitter_id, results|
+        if results && !( results.respond_to?( :length ) && results.length == 0 )
+          if block_given? 
+            continue = yield( twitter_id, results )
+          else
+            raise "You must supply a block to this method."
+          end
+          # keep fetching for this twitter_id only if the block said to and there are more updates.
+          twitter_ids << twitter_id if continue && !results.is_a?( TwitterException ) && results.length != 0
+        end
+      end
+    end
+    nil
+  end  
+
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'typho-twitter'
+require 'spec'
+require 'spec/autorun'
+
+module SpecLogger
+  def log message
+    puts message.to_s + "<br/>"
+  end
+end
+
+Spec::Runner.configure do |config|
+  
+end

data/test/oauth.yaml.example

@@ -0,0 +1,7 @@
+--- 
+:oauth: 
+  :consumer_key: CONSUMER_KEY
+  :consumer_secret: CONSUMER_SECRET
+  :secret: ACCESS_TOKEN_SECRET
+  :token: ACCESS_TOKEN
+  :site: http://example.com

data/test/typho-twitter-test.rb

@@ -0,0 +1,243 @@
+# A brutally simple test suite to verify desired functionality for +my+ application.
+# TODO: Generalize, and make unit tests.
+# NOTE: These tests are meant to be run against a white-listed account.  
+# To authorize with Twitter, you need to copy the oauth.yaml.example to oauth.yaml
+# and populate it with the OAuth credentials for a white-listed account.
+
+BASE_DIR="#{File.dirname(__FILE__)}/../lib"
+$: << BASE_DIR
+require 'typho-twitter'
+require 'test/unit'
+include Test::Unit::Assertions
+include WDD::Utils
+
+UNKNOWN_ID = 'mixtercox'
+TEST_USER_IDS = [ 
+  UNKNOWN_ID,
+  "bdoughty", 
+  "joshuabaer", 
+  "jotto", 
+  "hoonpark", 
+  "aplusk", 
+  "barackobama", 
+  "oprah",
+  "damon",
+  "TreyPlaysTunes",
+  "fiveredwoods",
+  "austinonrails",
+  "remlap42",
+  ]
+
+# basic test that all is functioning and that we can lookup an array of screen_names successfully
+def test_users_show_multi
+  screen_names = TEST_USER_IDS
+  screen_name_array = screen_names
+  # 10.times do |i|
+  #   screen_name_array += screen_names.map{|c| c+i}
+  # end
+  twitter = TyphoTwitter.new( @oauth_options )
+  # responses = twitter.typho_twitter_batch screen_name_array
+  responses = twitter.get_users_show( screen_name_array )
+  responses.each do |key, value|
+    puts "#{key} => "
+    if key == UNKNOWN_ID
+      assert_instance_of( TyphoTwitter::TwitterException, value )
+    end
+    puts "#{value}"
+  end
+  puts "# Responses: #{responses.size}"
+  assert_equal( responses.size, screen_name_array.size )
+end
+
+# Verify that things still function correctly if we are only looking up one user.
+def test_users_show_single
+  screen_names = [ 
+    "bdoughty"
+    ]
+  screen_name_array = screen_names
+  # screen_name_array += screen_names
+  twitter = TyphoTwitter.new( @oauth_options )
+  responses = twitter.get_users_show( screen_name_array )
+  responses.each do |key, value|
+    puts "#{key} => "
+    puts "#{value}"
+  end
+  puts "# Responses: #{responses.size}"
+  assert_equal( responses.size, screen_name_array.size )
+end
+
+# Test getting timelines passing a block to process_statuses_user_timeline.
+def test_process_statuses_user_timeline
+  screen_names = TEST_USER_IDS
+  screen_name_array = screen_names
+  # screen_name_array += screen_names
+  twitter = TyphoTwitter.new( @oauth_options )
+  user_updates = {}
+  twitter.process_statuses_user_timeline( screen_name_array ) do |screen_name, updates|
+    if screen_name == UNKNOWN_ID
+      assert_instance_of( TyphoTwitter::TwitterException, updates )
+      puts "Verified that #{UNKNOWN_ID} is unknown."
+      false
+    else
+      user_updates[screen_name] ||= []
+      user_updates[screen_name] += updates
+      true
+    end
+  end
+  user_updates.each do |key, value|
+    puts "#{key} => "
+    puts value.length
+  end
+  # puts "# Responses: #{responses.size}"
+  assert_equal( user_updates.size, screen_name_array.size - 1)
+end  
+
+# Test getting timelines from get_statuses_user_timeline
+def test_get_statuses_user_timeline
+  screen_names = TEST_USER_IDS
+  screen_name_array = screen_names
+  # screen_name_array += screen_names
+  twitter = TyphoTwitter.new( @oauth_options )
+  user_updates = twitter.get_statuses_user_timeline( screen_name_array )
+  user_updates.each do |key, value|
+    puts "#{key} => "
+    if key == UNKNOWN_ID
+      assert_instance_of( TyphoTwitter::TwitterException, value )
+      puts "Verified that #{UNKNOWN_ID} is unknown."
+    else
+      case value.class.to_s
+      when 'TyphoTwitter::TwitterException'
+        puts value.inspect
+      else
+        puts value.length
+      end
+    end
+    # pp value
+  end
+  # puts "# Responses: #{responses.size}"
+  assert_equal( user_updates.size, screen_name_array.size )
+end
+
+# Test getting followers
+def test_get_statuses_followers
+  # screen_names = ['hoonpark', 'bdoughty', 'jotto']
+  screen_names = ['bdoughty']
+  screen_name_array = screen_names
+  # screen_name_array += screen_names
+  twitter = TyphoTwitter.new( @oauth_options )
+  followers = twitter.get_statuses_followers( screen_name_array )
+  followers.each do |screen_name, results|
+    puts "#{screen_name}:"
+    puts "#{results.length} followers:"
+    results.each do |follower|
+      printvar :follower, follower
+    end
+    # pp value
+  end
+  assert_equal( followers.size,  screen_name_array.size )
+
+  screen_names = ['basdkhasdf']
+  screen_name_array = screen_names
+  # screen_name_array += screen_names
+  twitter = TyphoTwitter.new( @oauth_options )
+  followers = twitter.get_statuses_followers( screen_name_array )
+  followers.each do |screen_name, results|
+    puts "#{screen_name}:"
+    assert_instance_of TyphoTwitter::TwitterException, results
+  end
+end
+
+# Test getting followers
+def test_get_statuses_followers_with_limit
+  # screen_names = ['hoonpark', 'bdoughty', 'jotto']
+  screen_names = ['joshuabaer']
+  screen_name_array = screen_names
+  # screen_name_array += screen_names
+  twitter = TyphoTwitter.new( @oauth_options )
+  followers_data = twitter.get_statuses_followers( screen_name_array, 100 )
+  followers_data.each do |screen_name, followers|
+    puts "#{screen_name}:"
+    puts "#{followers.length} followers:"
+    followers.each do |follower|
+      printvar :follower, follower['screen_name']
+    end
+    assert_equal( followers.size <= 100,  true )
+    # pp value
+  end
+  assert_equal( followers_data.size,  screen_name_array.size )
+
+  screen_names = ['basdkhasdf']
+  screen_name_array = screen_names
+  # screen_name_array += screen_names
+  twitter = TyphoTwitter.new( @oauth_options )
+  followers = twitter.get_statuses_followers( screen_name_array )
+  followers.each do |screen_name, results|
+    puts "#{screen_name}:"
+    assert_instance_of TyphoTwitter::TwitterException, results
+  end
+end
+
+# Test getting timelines passing a block to process_statuses_user_timeline.  Process the updates and abort
+# on condition
+def test_process_statuses_user_timeline_with_abort
+  screen_names = TEST_USER_IDS
+  screen_name_array = screen_names
+  # screen_name_array += screen_names
+  twitter = TyphoTwitter.new( @oauth_options )
+  user_updates = {}
+  twitter.process_statuses_user_timeline( screen_name_array ) do |screen_name, results|
+    puts "#{screen_name} =>"
+    continue = true
+    if results.is_a? TyphoTwitter::TwitterException
+      puts "Exception for #{screen_name}: #{results.inspect}"
+      continue = false
+    else
+      results.each do |update|
+        # puts update["text"]
+        if update["text"] =~ /otherinbox/i
+          puts "ABORTING #{screen_name} because of tweet:"
+          puts update["text"]
+          continue = false
+          break
+        end
+      end
+      user_updates[screen_name] ||= []
+      user_updates[screen_name] += results
+    end
+    
+    continue
+  end
+  user_updates.each do |key, value|
+    puts "#{key} => "
+    puts value.length
+    # pp value
+  end
+  # puts "# Responses: #{responses.size}"
+  assert_equal( user_updates.size, screen_name_array.size - 1 )
+end  
+
+def time_it method_id
+  et = WDD::Utils::elapsed_time do
+    self.send( method_id )
+  end
+  puts "========================================"
+  puts "#{method_id.to_s} - #{et} seconds"
+  puts
+end
+
+def run_tests
+  @oauth_options = YAML.load_file(File.dirname($0)+'/oauth.yaml')
+  if ARGV[0] && ARGV[0] != ""
+    eval ARGV[0]
+  else
+    time_it :test_users_show_multi
+    time_it :test_users_show_single
+    time_it :test_process_statuses_user_timeline
+    time_it :test_get_statuses_user_timeline
+    time_it :test_get_statuses_followers
+    time_it :test_get_statuses_followers_with_limit
+    time_it :test_process_statuses_user_timeline_with_abort
+  end
+end
+
+run_tests

data/typho-twitter.gemspec

@@ -0,0 +1,63 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{typho-twitter}
+  s.version = "0.2.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["shock"]
+  s.date = %q{2011-01-16}
+  s.description = %q{A Twitter client for performing a batch of Twitter calls in parallel.}
+  s.email = %q{billdoughty@capitalthought.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/typho-twitter.rb",
+     "lib/typho-twitter/typho-twitter.rb",
+     "spec/spec.opts",
+     "spec/spec_helper.rb",
+     "test/oauth.yaml.example",
+     "test/typho-twitter-test.rb",
+     "typho-twitter.gemspec"
+  ]
+  s.homepage = %q{http://github.com/capitalthought/typho-twitter}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Parallel twitter client using Typhoeus.}
+  s.test_files = [
+    "spec/spec_helper.rb",
+     "test/typho-twitter-test.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_runtime_dependency(%q<wdd-ruby-ext>, [">= 0.3.1"])
+      s.add_runtime_dependency(%q<oauth>, [">= 0.4.3"])
+    else
+      s.add_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_dependency(%q<wdd-ruby-ext>, [">= 0.3.1"])
+      s.add_dependency(%q<oauth>, [">= 0.4.3"])
+    end
+  else
+    s.add_dependency(%q<rspec>, [">= 1.2.9"])
+    s.add_dependency(%q<wdd-ruby-ext>, [">= 0.3.1"])
+    s.add_dependency(%q<oauth>, [">= 0.4.3"])
+  end
+end
+

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification 
+name: typho-twitter
+version: !ruby/object:Gem::Version 
+  hash: 21
+  prerelease: false
+  segments: 
+  - 0
+  - 2
+  - 1
+  version: 0.2.1
+platform: ruby
+authors: 
+- shock
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-16 00:00:00 -06:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: wdd-ruby-ext
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 17
+        segments: 
+        - 0
+        - 3
+        - 1
+        version: 0.3.1
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: oauth
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 4
+        - 3
+        version: 0.4.3
+  type: :runtime
+  version_requirements: *id003
+description: A Twitter client for performing a batch of Twitter calls in parallel.
+email: billdoughty@capitalthought.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/typho-twitter.rb
+- lib/typho-twitter/typho-twitter.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- test/oauth.yaml.example
+- test/typho-twitter-test.rb
+- typho-twitter.gemspec
+has_rdoc: true
+homepage: http://github.com/capitalthought/typho-twitter
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Parallel twitter client using Typhoeus.
+test_files: 
+- spec/spec_helper.rb
+- test/typho-twitter-test.rb