devfu-twitter-search-watcher 0.1.1 → 0.1.2

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/lib/twitter-search-watcher.rb

@@ -1,5 +1,10 @@
 %w( rubygems cgi json open-uri ostruct ).each {|lib| require lib }
 
+# 
+# = Usage
+#
+#   coming soon
+#
 class TwitterSearchWatcher
 
   TWITTER_SEARCH_URL      = 'http://search.twitter.com/search.json'
@@ -87,7 +92,7 @@ class TwitterSearchWatcher
 
   # Performs a search.  Accepts the same parameters as #search_url
   def search! additional_parameters = nil
-    JSON.parse open( search_url(additional_parameters), 'User-Agent' => user_agent ).read
+    json search_url(additional_parameters)
   end
 
   # Performs a search, given the response from another search.
@@ -114,6 +119,68 @@ class TwitterSearchWatcher
     search!( (additional_parameters || {}).merge( :page => (response['page'] + 1), :max_id => response['max_id'] ) ) if response['next_page']
   end
 
+  # Performs a search! and search_more! as needed to return a response with *all* pages of tweets.
+  #
+  # This respects max_pages and will only make max_pages number of additional requests to get paginated tweets.
+  #
+  # The response object returned is similar to the responses returned by all other methods, but we only 
+  # currently have a 'results' key on the Hash returned.  If you're used to getting some of the other keys 
+  # returned by the other methods (which Twitter returns), be warned!
+  #
+  # To get the tweets off of the response:
+  #
+  #   tweets = watcher.search_with_pagination!['results']
+  def search_with_pagination! additional_parameters = nil
+    response     = search! additional_parameters
+
+    max_requests = max_pages
+    max_requests = additional_parameters[:max_pages] if additional_parameters && additional_parameters[:max_pages]
+
+    tweets = { 'results' => [] }
+    pages_requested_so_far = 0
+
+    if response['next_page']
+      while response = search_more!(response)
+        tweets['results'] << response['results']
+        pages_requested_so_far += 1
+
+        break if max_requests && pages_requested_so_far >= max_requests
+      end
+    end
+
+    tweets
+  end
+
+  # Helper to do an HTTP GET request and return the response body
+  def get url
+    TwitterSearchWatcher.get url, 'User-Agent' => user_agent
+  end
+
+  # Helper to do an HTTP GET request and return the response body
+  def self.get url, options = {}
+    open( url, { 'User-Agent' => DEFAULT_USER_AGENT }.merge(options) ).read
+  end
+
+  # Helper to #get a url and return the response body parsed as JSON
+  def json url
+    JSON.parse get(url)
+  end
+
+  # Helper to #get a url and return the response body parsed as JSON
+  def self.json url
+    JSON.parse get(url)
+  end
+
+  # Instantiates a new TwitterSearchWatcher given the search_string and options and then 
+  # calls search_with_pagination! on the instance, returning the response.
+  #
+  #   tweets_json = TwitterSearchWatcher.search_with_pagination!('foo')['results']
+  #
+  def self.search_with_pagination! search_string, options = nil
+    watcher = TwitterSearchWatcher.new search_string, options
+    watcher.search_with_pagination!
+  end
+
   # Instantiates a new TwitterSearchWatcher given the search_string and options and then 
   # calls #watch on the instance using the block given.
   def self.watch! search_string, options = nil, &block

data/spec/search_spec.rb

@@ -0,0 +1,131 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe TwitterSearchWatcher, 'searching' do
+
+  it 'should be able to execute search' do
+    watcher = TSW.new 'chunky bacon'
+    TSW.should_receive(:get).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response)
+    response = watcher.search!
+    response.should be_a_kind_of(Hash)
+    response['query'].should == 'remitaylor' # make sure it's getting our fake date
+  end
+
+  it 'should be able to search!(:rpp => 15) ... inotherwords, should be able to pass additional parameters to search!' do
+    watcher = TSW.new 'foo'
+
+    TSW.should_receive(:get).with(/99/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
+    watcher.search! :rpp => 99
+
+    TSW.should_receive(:get).with(/foo/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
+    watcher.search! :rpp => 99
+  end
+
+  it 'should be able to search!(:rpp => nil) to override an existing parameter' do
+    watcher = TSW.new 'foo', :rpp => 50
+    watcher.search_url.should include('rpp=50')
+    watcher.search_url.should include('q=foo')
+
+    # double check that normal overrides are working properly
+    watcher.search_url( :rpp => 99 ).should_not include('rpp=50')
+    watcher.search_url( :rpp => 99 ).should include('rpp=99')
+
+    watcher.search_url( :rpp => nil ).should_not include('rpp=50')
+    watcher.search_url( :rpp => nil ).should_not include('rpp')
+  end
+
+  it 'should be able to search_newer!(result_set) to get results newer than a given result set' do
+    watcher = TSW.new 'chunky bacon'
+    TSW.should_receive(:get).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response(:max_id => 5555))
+    response = watcher.search!
+
+    TSW.should_receive(:get).with(/since_id=5555/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
+    watcher.search_newer! response
+  end
+
+  it 'should be able to search_more!(result_set) to get paginated results' do
+    with_next_page    = fake_response :page => 1, :max_id => 1, :next_page => '?page=2&max_id=444&q=chunky+bacon'
+    without_next_page = fake_response :page => 1, :max_id => 2, :next_page => nil
+
+    watcher = TSW.new 'chunky bacon'
+
+    TSW.should_receive(:get).once.with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(with_next_page)
+    response = watcher.search!
+    
+    TSW.should_receive(:get).once.with(/page=2/, 'User-Agent' => watcher.user_agent).and_return(without_next_page)
+    response2 = watcher.search_more! response
+
+    watcher.should_not_receive(:open)
+    watcher.search_more!( response2 ).should be_nil # because no next_page
+  end
+
+  it 'should be able to watch! on a watcher' do
+    # it's a PITA to test the loop, so i'm not currently testing that check_every and max_pages are actually used correctly
+    watcher = TSW.new 'chunky bacon', :check_every => 120, :max_pages => 5
+    watcher.check_every.should == 120
+    watcher.max_pages.should == 5
+    watcher.should respond_to(:watch!)
+  end
+
+  it 'should be able to TSW.search_with_pagination!' do
+    _2_pages_left = fake_response :page => 1, :max_id => 1, :next_page => '?page=2&max_id=1&q=foo'
+    _1_page_left  = fake_response :page => 2, :max_id => 2, :next_page => '?page=3&max_id=1&q=foo'
+    _0_pages_left = fake_response :page => 3, :max_id => 3, :next_page => nil
+
+    TSW.should_receive(:get).once.with(/q=foo/,  anything).and_return(_2_pages_left)
+    TSW.should_receive(:get).once.with(/page=2/, anything).and_return(_1_page_left)
+    TSW.should_receive(:get).once.with(/page=3/, anything).and_return(_0_pages_left)
+
+    TSW.search_with_pagination! 'foo'
+  end
+
+  it 'should be able to search_with_pagination!' do
+    _2_pages_left = fake_response :page => 1, :max_id => 1, :next_page => '?page=2&max_id=1&q=foo'
+    _1_page_left  = fake_response :page => 2, :max_id => 2, :next_page => '?page=3&max_id=1&q=foo'
+    _0_pages_left = fake_response :page => 3, :max_id => 3, :next_page => nil
+
+    watcher = TSW.new 'foo'
+
+    TSW.should_receive(:get).once.with(/q=foo/,  anything).and_return(_2_pages_left)
+    TSW.should_receive(:get).once.with(/page=2/, anything).and_return(_1_page_left)
+    TSW.should_receive(:get).once.with(/page=3/, anything).and_return(_0_pages_left)
+
+    watcher.search_with_pagination!
+  end
+
+  it 'search_with_pagination! should respect max_pages (instance)' do
+    _2_pages_left = fake_response :page => 1, :max_id => 1, :next_page => '?page=2&max_id=1&q=foo'
+    _1_page_left  = fake_response :page => 2, :max_id => 2, :next_page => '?page=3&max_id=1&q=foo'
+    _0_pages_left = fake_response :page => 3, :max_id => 3, :next_page => nil
+
+    watcher = TSW.new 'foo', :max_pages => 1 # it'll only get 1 page *after* doing the original search! request
+
+    TSW.should_receive(:get).once.with(/q=foo/, 'User-Agent' => watcher.user_agent).and_return(_2_pages_left)
+    TSW.should_receive(:get).once.with(/page=2/, 'User-Agent' => watcher.user_agent).and_return(_1_page_left)
+    watcher.should_not_receive(:open)
+
+    watcher.search_with_pagination!
+  end
+
+  it 'search_with_pagination! should respect max_pages (override)' do
+    _2_pages_left = fake_response :page => 1, :max_id => 1, :next_page => '?page=2&max_id=1&q=foo'
+    _1_page_left  = fake_response :page => 2, :max_id => 2, :next_page => '?page=3&max_id=1&q=foo'
+    _0_pages_left = fake_response :page => 3, :max_id => 3, :next_page => nil
+
+    watcher = TSW.new 'foo', :max_pages => 99
+
+    TSW.should_receive(:get).once.with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(_2_pages_left)
+    TSW.should_receive(:get).once.with(/page=2/, 'User-Agent' => watcher.user_agent).and_return(_1_page_left)
+    watcher.should_not_receive(:open)
+
+    watcher.search_with_pagination! :max_pages => 1
+  end
+
+  # Ideas ...
+
+  #it 'should be able to search! (with pagination)'
+  #it 'should default to getting additional pages when doing a search!'
+  #it 'default rpp should be 100 (the max)'
+  #it 'should have a nice method to call that will make a new watcher and start watching'
+  #it 'should be able to watch! with pagination (will search! while there is a :next_page to request)'
+
+end

data/spec/search_url_generation_spec.rb

@@ -0,0 +1,72 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe TwitterSearchWatcher, '#search_url generation' do
+
+  it 'should be able to initialize with a search string' do
+    TwitterSearchWatcher.new('foo').search_url.should == 'http://search.twitter.com/search.json?q=foo'
+    TwitterSearchWatcher.new('#foo').search_url.should == 'http://search.twitter.com/search.json?q=%23foo'
+    TwitterSearchWatcher.new('@foo').search_url.should == 'http://search.twitter.com/search.json?q=%40foo'
+  end
+
+  it 'should be able to search for replies :to someone' do
+    TwitterSearchWatcher.new( :to => 'remitaylor' ).search_url.should end_with('search.json?to=remitaylor')
+  end
+
+  it 'should be able to search for tweets :from someone' do
+    TwitterSearchWatcher.new( :from => 'remitaylor' ).search_url.should end_with('search.json?from=remitaylor')
+  end
+
+  it 'should be able to search for tweets :from someone that are :to someone' do
+    watcher = TwitterSearchWatcher.new :from => 'remitaylor', :to => 'wickd_wanda'
+    watcher.search_url.should include('from=remitaylor')
+    watcher.search_url.should include('to=wickd_wanda')
+  end
+
+  it 'should have a good default User-Agent' do
+    watcher = TwitterSearchWatcher.new
+    watcher.user_agent.should == 'TwitterSearchWatcher RubyGem http://github.com/devfu/twitter-search-watcher'
+    watcher.user_agent = 'foo'
+    watcher.user_agent.should == 'foo'
+  end
+
+  it 'should be able to override User-Agent' do
+    watcher = TwitterSearchWatcher.new 'x', :user_agent => 'bar'
+    watcher.search_url.should == 'http://search.twitter.com/search.json?q=x'
+    watcher.user_agent.should == 'bar'
+
+    watcher = TwitterSearchWatcher.new :user_agent => 'bar'
+    watcher.search_url.should == 'http://search.twitter.com/search.json?'
+    watcher.user_agent.should == 'bar'
+  end
+
+  it 'should allow quotes in query string' do
+    TwitterSearchWatcher.new('"hello there"').search_url.should end_with('q="hello+there"')
+  end
+
+  it 'should replace all spaces in query string with + signs' do
+    TwitterSearchWatcher.new('hello there').search_url.should end_with('q=hello+there')
+  end
+
+  it 'should be able to request a particular :page (pagination)' do
+    TwitterSearchWatcher.new( 'foo', :page => 2 ).search_url.should include("page=2")
+    TwitterSearchWatcher.new( 'foo', :page => 2 ).search_url.should include("q=foo")
+  end
+
+  it 'should be able to request since a particular twitter ID (:since_id)' do
+    TwitterSearchWatcher.new( 'foo', :since_id => 1234 ).search_url.should include("since_id=1234")
+    TwitterSearchWatcher.new( 'foo', :since_id => 1234 ).search_url.should include("q=foo")
+  end
+
+  it 'should be able to request with a particular :max_id (used with :page)' do
+    TwitterSearchWatcher.new( 'foo', :max_id => 4321 ).search_url.should include("max_id=4321")
+    TwitterSearchWatcher.new( 'foo', :max_id => 4321 ).search_url.should include("q=foo")
+  end
+
+  it 'should be able to request with a particular :rpp (results_per_page)' do
+    TwitterSearchWatcher.new( 'foo', :rpp => 90 ).search_url.should include("rpp=90")
+    TwitterSearchWatcher.new( 'foo', :rpp => 90 ).search_url.should include("q=foo")
+
+    lambda { TwitterSearchWatcher.new( 'foo', :rpp => 101 ) }.should raise_error(/100/)
+  end
+
+end

data/spec/spec_helper.rb

@@ -2,6 +2,9 @@ require File.dirname(__FILE__) + '/../lib/twitter-search-watcher'
 require 'rubygems'
 require 'spec'
 require 'ostruct'
+require 'activesupport' # for stringify_keys
+
+TSW = TwitterSearchWatcher
 
 def readable value
   OpenStruct.new({ :read => value })
@@ -12,20 +15,18 @@ def fake_response_for watcher
 end
 
 def fake_response options = {}
-  readable(
-    {
-      'max_id'           => 1234,
-      'since_id'         => 0,
-      'total'            => 15, # doesn't appear if next_page present?
-      'next_page'        => '?page=2&max_id=1234&q=remitaylor', # only appears if there are additional pages
-      'refresh_url'      => '?since_id=1234&q=remitaylor',
-      'page'             => 1,
-      'results_per_page' => 15,
-      'completed_in'     => 0.1234,
-      'query'            => 'remitaylor',
-      'results'          => fake_tweets
-    }.merge(options).to_json
-  )
+  {
+    'max_id'           => 1234,
+    'since_id'         => 0,
+    'total'            => 15, # doesn't appear if next_page present?
+    'next_page'        => '?page=2&max_id=1234&q=remitaylor', # only appears if there are additional pages
+    'refresh_url'      => '?since_id=1234&q=remitaylor',
+    'page'             => 1,
+    'results_per_page' => 15,
+    'completed_in'     => 0.1234,
+    'query'            => 'remitaylor',
+    'results'          => fake_tweets
+  }.merge(options.stringify_keys).to_json
 end
 
 def fake_tweets

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: devfu-twitter-search-watcher
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - remi
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-06 00:00:00 -07:00
+date: 2009-09-04 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -26,9 +26,10 @@ files:
 - Rakefile
 - VERSION
 - lib/twitter-search-watcher.rb
+- spec/search_spec.rb
+- spec/search_url_generation_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-- spec/twitter_search_watcher_spec.rb
 has_rdoc: false
 homepage: http://github.com/devfu/twitter-search-watcher
 licenses: 
@@ -57,5 +58,6 @@ signing_key:
 specification_version: 3
 summary: for watching a Twitter search
 test_files: 
-- spec/twitter_search_watcher_spec.rb
+- spec/search_spec.rb
 - spec/spec_helper.rb
+- spec/search_url_generation_spec.rb

data/spec/twitter_search_watcher_spec.rb

@@ -1,140 +0,0 @@
-require File.dirname(__FILE__) + '/spec_helper'
-
-describe TwitterSearchWatcher do
-
-  it 'should be able to initialize with a search string' do
-    TwitterSearchWatcher.new('foo').search_url.should == 'http://search.twitter.com/search.json?q=foo'
-    TwitterSearchWatcher.new('#foo').search_url.should == 'http://search.twitter.com/search.json?q=%23foo'
-    TwitterSearchWatcher.new('@foo').search_url.should == 'http://search.twitter.com/search.json?q=%40foo'
-  end
-
-  it 'should be able to search for replies :to someone' do
-    TwitterSearchWatcher.new( :to => 'remitaylor' ).search_url.should end_with('search.json?to=remitaylor')
-  end
-
-  it 'should be able to search for tweets :from someone' do
-    TwitterSearchWatcher.new( :from => 'remitaylor' ).search_url.should end_with('search.json?from=remitaylor')
-  end
-
-  it 'should be able to search for tweets :from someone that are :to someone' do
-    watcher = TwitterSearchWatcher.new :from => 'remitaylor', :to => 'wickd_wanda'
-    watcher.search_url.should include('from=remitaylor')
-    watcher.search_url.should include('to=wickd_wanda')
-  end
-
-  it 'should have a good default User-Agent' do
-    watcher = TwitterSearchWatcher.new
-    watcher.user_agent.should == 'TwitterSearchWatcher RubyGem http://github.com/devfu/twitter-search-watcher'
-    watcher.user_agent = 'foo'
-    watcher.user_agent.should == 'foo'
-  end
-
-  it 'should be able to override User-Agent' do
-    watcher = TwitterSearchWatcher.new 'x', :user_agent => 'bar'
-    watcher.search_url.should == 'http://search.twitter.com/search.json?q=x'
-    watcher.user_agent.should == 'bar'
-
-    watcher = TwitterSearchWatcher.new :user_agent => 'bar'
-    watcher.search_url.should == 'http://search.twitter.com/search.json?'
-    watcher.user_agent.should == 'bar'
-  end
-
-  it 'should allow quotes in query string' do
-    TwitterSearchWatcher.new('"hello there"').search_url.should end_with('q="hello+there"')
-  end
-
-  it 'should replace all spaces in query string with + signs' do
-    TwitterSearchWatcher.new('hello there').search_url.should end_with('q=hello+there')
-  end
-
-  it 'should be able to execute search' do
-    watcher = TwitterSearchWatcher.new 'chunky bacon'
-    watcher.should_receive(:open).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response)
-    response = watcher.search!
-    response.should be_a_kind_of(Hash)
-    response['query'].should == 'remitaylor' # make sure it's getting our fake date
-  end
-
-  it 'should be able to request a particular :page (pagination)' do
-    TwitterSearchWatcher.new( 'foo', :page => 2 ).search_url.should include("page=2")
-    TwitterSearchWatcher.new( 'foo', :page => 2 ).search_url.should include("q=foo")
-  end
-
-  it 'should be able to request since a particular twitter ID (:since_id)' do
-    TwitterSearchWatcher.new( 'foo', :since_id => 1234 ).search_url.should include("since_id=1234")
-    TwitterSearchWatcher.new( 'foo', :since_id => 1234 ).search_url.should include("q=foo")
-  end
-
-  it 'should be able to request with a particular :max_id (used with :page)' do
-    TwitterSearchWatcher.new( 'foo', :max_id => 4321 ).search_url.should include("max_id=4321")
-    TwitterSearchWatcher.new( 'foo', :max_id => 4321 ).search_url.should include("q=foo")
-  end
-
-  it 'should be able to request with a particular :rpp (results_per_page)' do
-    TwitterSearchWatcher.new( 'foo', :rpp => 90 ).search_url.should include("rpp=90")
-    TwitterSearchWatcher.new( 'foo', :rpp => 90 ).search_url.should include("q=foo")
-
-    lambda { TwitterSearchWatcher.new( 'foo', :rpp => 101 ) }.should raise_error(/100/)
-  end
-
-  it 'should be able to search!(:rpp => 15) ... inotherwords, should be able to pass additional parameters to search!' do
-    watcher = TwitterSearchWatcher.new 'foo'
-
-    watcher.should_receive(:open).with(/99/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
-    watcher.search! :rpp => 99
-
-    watcher.should_receive(:open).with(/foo/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
-    watcher.search! :rpp => 99
-  end
-
-  it 'should be able to search!(:rpp => nil) to override an existing parameter' do
-    watcher = TwitterSearchWatcher.new 'foo', :rpp => 50
-    watcher.search_url.should include('rpp=50')
-    watcher.search_url.should include('q=foo')
-
-    # double check that normal overrides are working properly
-    watcher.search_url( :rpp => 99 ).should_not include('rpp=50')
-    watcher.search_url( :rpp => 99 ).should include('rpp=99')
-
-    watcher.search_url( :rpp => nil ).should_not include('rpp=50')
-    watcher.search_url( :rpp => nil ).should_not include('rpp')
-  end
-
-  it 'should be able to search_newer!(result_set) to get results newer than a given result set' do
-    watcher = TwitterSearchWatcher.new 'chunky bacon'
-    watcher.should_receive(:open).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response(:max_id => 5555))
-    response = watcher.search!
-
-    watcher.should_receive(:open).with(/since_id=5555/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
-    watcher.search_newer! response
-  end
-
-  it 'should be able to search_more!(result_set) to get paginated results' do
-    watcher = TwitterSearchWatcher.new 'chunky bacon'
-    watcher.should_receive(:open).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response(:page => 1, :max_id => 444, :next_page => '?page=2&max_id=444&q=chunky+bacon'))
-    response = watcher.search!
-    
-    watcher.should_receive(:open).with(/page=2/, 'User-Agent' => watcher.user_agent).and_return(fake_response(:page => 2, :max_id => 445, :next_page => nil))
-    response2 = watcher.search_more! response
-    watcher.should_receive(:open).with(/max_id=444/, 'User-Agent' => watcher.user_agent).and_return(fake_response(:page => 2, :max_id => 445, :next_page => nil))
-    response2 = watcher.search_more! response
-
-    watcher.should_not_receive(:open)
-    watcher.search_more!( response2 ).should be_nil # because no next_page
-  end
-
-  it 'should be able to watch! on a watcher' do
-    # it's a PITA to test the loop, so i'm not currently testing that check_every and max_pages are actually used correctly
-    watcher = TwitterSearchWatcher.new 'chunky bacon', :check_every => 120, :max_pages => 5
-    watcher.check_every.should == 120
-    watcher.max_pages.should == 5
-    watcher.should respond_to(:watch!)
-  end
-
-  it 'default rpp should be 100 (the max)'
-
-  it 'should have a nice method to call that will make a new watcher and start watching'
-
-  it 'should be able to watch! with pagination (will search! while there is a :next_page to request)'
-
-end