devfu-twitter-search-watcher 0.1.0

data/README.rdoc

@@ -0,0 +1,21 @@
+= TwitterSearchWatcher
+
+Sometimes, we want to continually search Twitter for something 
+like a hashtag to display the latest results on our sites.
+
+TwitterSearchWatcher is a simple gem for continually searching 
+Twitter for something and calling a callback whenever new 
+tweets are present.
+
+== Installation
+
+  sudo gem install json
+  sudo gem install devfu-twitter-search-watcher -s http://gems.github.com
+
+== Usage
+
+  TwitterSearchWatcher.watch! '#laidoffcamp', :max_pages => 1 do |tweet|
+    puts "#{ tweet.from_user }: #{ tweet.text }"
+  end
+
+For additional usage, see RDoc at http://code.devfu.com/twitter-search-watcher or the specs at http://github.com/devfu/twitter-search-watcher/blob/master/spec/twitter_search_watcher_spec.rb

data/Rakefile

@@ -0,0 +1,68 @@
+require 'rake'
+require 'rubygems'
+require 'rake/rdoctask'
+require 'spec/rake/spectask'
+
+puts "\nGem: twitter-search-watcher\n\n"
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name        = 'twitter-search-watcher'
+    s.summary     = 'for watching a Twitter search'
+    s.email       = 'remi@remitaylor.com'
+    s.homepage    = 'http://github.com/devfu/twitter-search-watcher'
+    s.description = 'for watching a particular Twitter search and calling code whenever there are new tweets'
+    s.authors     = %w( remi )
+    s.files       = FileList['[A-Z]*', '{lib,spec,bin,examples}/**/*'] 
+    # s.add_dependency 'person-gemname'
+    # s.executables << 'script'
+    # s.rubyforge_project = 'gemname'
+    # s.extra_rdoc_files = %w( README.rdoc )
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+Spec::Rake::SpecTask.new do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+desc "Run all examples with RCov"
+Spec::Rake::SpecTask.new('rcov') do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.rcov = true
+end
+
+# require 'hanna'
+# require 'darkfish-rdoc'
+
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'twitter-search-watcher'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  # rdoc.options += ["--template=#{`allison --path`}"]  # sudo gem install allison
+  # rdoc.options += %w( -f darkfish )                   # sudo gem install darkfish-rdoc
+  # rdoc.options += %w( -T hanna )                      # sudo gem install mislav-hanna
+  rdoc.options += %w( -m README.rdoc ) # the initial page displayed
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+desc 'Confirm that gemspec is $SAFE'
+task :safe do
+  require 'yaml'
+  require 'rubygems/specification'
+  data = File.read('twitter-search-watcher.gemspec')
+  spec = nil
+  if data !~ %r{!ruby/object:Gem::Specification}
+    Thread.new { spec = eval("$SAFE = 3\n#{data}") }.join
+  else
+    spec = YAML.load(data)
+  end
+  spec.validate
+  puts spec
+  puts "OK"
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/twitter-search-watcher.rb

@@ -0,0 +1,191 @@
+%w( rubygems cgi json open-uri ostruct ).each {|lib| require lib }
+
+class TwitterSearchWatcher
+
+  TWITTER_SEARCH_URL      = 'http://search.twitter.com/search.json'
+  DEFAULT_USER_AGENT      = 'TwitterSearchWatcher RubyGem http://github.com/devfu/twitter-search-watcher'
+  QUERY_STRING_ATTRIBUTES = [ :q, :to, :from, :since_id, :page, :max_id, :rpp ]
+
+  # The User-Agent header value to send along with all Twitter Search API requests
+  attr_accessor :user_agent
+
+  # A string you want to search twitter for
+  attr_accessor :q
+  
+  # The username of someone you want to search replies to
+  attr_accessor :to
+
+  # The username of someone you want to search replies from
+  attr_accessor :from
+
+  # Get a particular page of Twitter search results (pagination).
+  # Typically used in conjunction with :max_id
+  attr_accessor :page
+
+  # Used for pagination, so you can get page=3 where the max_id of the first page was 1234
+  attr_accessor :max_id
+
+  # Only get tweets with ID's greater than this ID (useful for only getting new tweets)
+  attr_accessor :since_id
+
+  # Number of results per page (max 100)
+  attr_accessor :rpp
+
+  # The number of seconds to wait between Twitter calls.  Default: 60 (seconds)
+  attr_accessor :check_every
+
+  # The maximum number of pages to check for tweets
+  #
+  # If nil, we'll check until there are no more pages (when :next_page isn't present)
+  attr_accessor :max_pages
+
+  def rpp= value
+    raise "The maximum rpp (Results per Page) value is 100" if value > 100
+    @rpp = value
+  end
+
+  def check_every
+    @check_every || 60
+  end
+
+  # Create a new TwitterSearchWatcher
+  #
+  #   TwitterSearchWatcher.new 'string to search'
+  #   TwitterSearchWatcher.new 'string to search', :check_every => 60
+  #   TwitterSearchWatcher.new :to => 'barackobama', :from => 'SenJohnMcCain'
+  #
+  def initialize search_string = nil, options = nil
+    if search_string.is_a? Hash
+      options = search_string
+    else
+      self.q = search_string
+    end
+
+    options.each {|k,v| send "#{k}=", v } if options
+  end
+
+  # Returns the URL we'll use to call the Twitter Search API.
+  #
+  # Without parameters, it'll generate a URL just from this TwitterSearchWatcher instance.
+  #
+  # With parameters, it'll override the TwitterSearchWatcher instance's options with 
+  # whatever you pass, eg.
+  #
+  #   >> TwitterSearchWatcher.new( 'foo', :rpp => 15 ).search_url
+  #   => "http://search.twitter.com/search.json?q=foo&rpp=15"
+  #
+  #   >> TwitterSearchWatcher.new( 'foo', :rpp => 15 ).search_url( :rpp => 99 )
+  #   => "http://search.twitter.com/search.json?q=foo&rpp=99"
+  #
+  def search_url additional_parameters = nil
+    TWITTER_SEARCH_URL + build_query_string(additional_parameters)
+  end
+
+  def user_agent
+    @user_agent || DEFAULT_USER_AGENT
+  end
+
+  # Performs a search.  Accepts the same parameters as #search_url
+  def search! additional_parameters = nil
+    JSON.parse open( search_url(additional_parameters), 'User-Agent' => user_agent ).read
+  end
+
+  # Performs a search, given the response from another search.
+  #
+  # If a response if given, the search will only return tweets newer than the given response's tweets.
+  # If a response is not given, this performs a normal search.
+  #
+  # Accepts additional parameters (same as #search_url)
+  def search_newer! response = nil, additional_parameters = nil
+    if response
+      search!( (additional_parameters || {}).merge( :since_id => response['max_id'] ) )
+    else
+      search! additional_parameters
+    end
+  end
+
+  # Performs a search, given the response from another search.
+  #
+  # If the response given is paginated (ie. there are additional tweets available on additional pages), 
+  # this will return the next page.  Else, this will return nil.
+  #
+  # Accepts additional parameters (same as #search_url)
+  def search_more! response, additional_parameters = nil
+    search!( (additional_parameters || {}).merge( :page => (response['page'] + 1), :max_id => response['max_id'] ) ) if response['next_page']
+  end
+
+  # Instantiates a new TwitterSearchWatcher given the search_string and options and then 
+  # calls #watch on the instance using the block given.
+  def self.watch! search_string, options = nil, &block
+    watcher = TwitterSearchWatcher.new search_string, options
+    watcher.watch! &block
+  end
+
+  # Starts watching this search in a loop.
+  # It will wait #check_every seconds between new requests (except requests to get additional pages).
+  # Every time a new tweet is found, that tweet is passed to the block given.
+  #
+  #   TwitterSearchWatcher.new('foo').watch! {|tweet| puts "got tweet: #{ tweet.text }" }
+  #
+  def watch! additional_parameters = nil, &block
+    @max_id_found_so_far = 0
+  
+    trap('INT'){ puts "\nexiting ..."; exit }
+    puts "Watching for tweets: #{ search_url(additional_parameters) }"
+
+    loop do
+
+      @last_response = search_newer!(@last_response, additional_parameters)
+      call_tweet_callbacks(@last_response, block)
+      update_max_id @last_response
+
+      # this is kindof icky ... but it works
+      if @last_response['next_page']
+        response = @last_response
+        num_pages_searched = 0
+        while (response = search_more!(response, additional_parameters)) && (num_pages_searched <= max_pages if max_pages)
+          num_pages_searched += 1
+          call_tweet_callbacks(response, block)
+          update_max_id response
+        end
+      end
+
+      sleep check_every
+    end
+  end
+
+private
+
+  def update_max_id response
+    @max_id_found_so_far = response['max_id'] if response['max_id'] > @max_id_found_so_far
+  end
+
+  def call_tweet_callbacks response, block
+    response['results'].each do |tweet|
+      block.call OpenStruct.new(tweet)
+    end
+  end
+
+  def escape string
+    CGI.escape(string.to_s).gsub('%22','"').gsub(' ','+')
+  end
+
+  def build_query_string additional_parameters = nil
+    parameter_values = QUERY_STRING_ATTRIBUTES.inject({}){|all, attr|
+      all[attr] = send(attr) if send(attr)
+      all
+    }
+
+    # if additional parameters are passed, we override the watcher's parameters with these
+    if additional_parameters
+      additional_parameter_values = QUERY_STRING_ATTRIBUTES.inject({}){|all, attr|
+        all[attr] = additional_parameters[attr] if additional_parameters.keys.include?(attr)
+        all
+      }
+      parameter_values.merge! additional_parameter_values
+    end
+
+    '?' + parameter_values.map {|k,v| "#{ k }=#{ escape(v) }" if v }.compact.join('&')
+  end
+
+end

data/spec/spec.opts

@@ -0,0 +1,4 @@
+--colour
+--format specdoc
+--loadby mtime
+--reverse

data/spec/spec_helper.rb

@@ -0,0 +1,53 @@
+require File.dirname(__FILE__) + '/../lib/twitter-search-watcher'
+require 'rubygems'
+require 'spec'
+require 'ostruct'
+
+def readable value
+  OpenStruct.new({ :read => value })
+end
+
+def fake_response_for watcher
+
+end
+
+def fake_response options = {}
+  readable(
+    {
+      'max_id'           => 1234,
+      'since_id'         => 0,
+      'total'            => 15, # doesn't appear if next_page present?
+      'next_page'        => '?page=2&max_id=1234&q=remitaylor', # only appears if there are additional pages
+      'refresh_url'      => '?since_id=1234&q=remitaylor',
+      'page'             => 1,
+      'results_per_page' => 15,
+      'completed_in'     => 0.1234,
+      'query'            => 'remitaylor',
+      'results'          => fake_tweets
+    }.merge(options).to_json
+  )
+end
+
+def fake_tweets
+  [ fake_tweet, fake_tweet ]
+end
+
+def fake_tweet
+  {
+    "created_at"=>"Wed, 05 Aug 2009 03:54:03 +0000", 
+    "profile_image_url"=>"http://s3.amazonaws.com/twitter_production/profile_images/346217637/tiltshiftdino_normal.jpg", 
+    "from_user"=>"BaddMann", 
+    "to_user_id"=>549282, 
+    "text"=>"@remitaylor What did you think of the badges at defcon Did you try anything with your badge", 
+    "id" => (rand * 1000000000).to_i, 
+    "from_user_id"=>160309, 
+    "to_user"=>"remitaylor", 
+    "iso_language_code"=>"en", 
+    "source"=>"<a href="http://twitter.com/">web</a>"}
+end
+
+Spec::Matchers.define :end_with do |expected|
+  match do |actual|
+    actual.end_with? expected
+  end
+end

data/spec/twitter_search_watcher_spec.rb

@@ -0,0 +1,140 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe TwitterSearchWatcher do
+
+  it 'should be able to initialize with a search string' do
+    TwitterSearchWatcher.new('foo').search_url.should == 'http://search.twitter.com/search.json?q=foo'
+    TwitterSearchWatcher.new('#foo').search_url.should == 'http://search.twitter.com/search.json?q=%23foo'
+    TwitterSearchWatcher.new('@foo').search_url.should == 'http://search.twitter.com/search.json?q=%40foo'
+  end
+
+  it 'should be able to search for replies :to someone' do
+    TwitterSearchWatcher.new( :to => 'remitaylor' ).search_url.should end_with('search.json?to=remitaylor')
+  end
+
+  it 'should be able to search for tweets :from someone' do
+    TwitterSearchWatcher.new( :from => 'remitaylor' ).search_url.should end_with('search.json?from=remitaylor')
+  end
+
+  it 'should be able to search for tweets :from someone that are :to someone' do
+    watcher = TwitterSearchWatcher.new :from => 'remitaylor', :to => 'wickd_wanda'
+    watcher.search_url.should include('from=remitaylor')
+    watcher.search_url.should include('to=wickd_wanda')
+  end
+
+  it 'should have a good default User-Agent' do
+    watcher = TwitterSearchWatcher.new
+    watcher.user_agent.should == 'TwitterSearchWatcher RubyGem http://github.com/devfu/twitter-search-watcher'
+    watcher.user_agent = 'foo'
+    watcher.user_agent.should == 'foo'
+  end
+
+  it 'should be able to override User-Agent' do
+    watcher = TwitterSearchWatcher.new 'x', :user_agent => 'bar'
+    watcher.search_url.should == 'http://search.twitter.com/search.json?q=x'
+    watcher.user_agent.should == 'bar'
+
+    watcher = TwitterSearchWatcher.new :user_agent => 'bar'
+    watcher.search_url.should == 'http://search.twitter.com/search.json?'
+    watcher.user_agent.should == 'bar'
+  end
+
+  it 'should allow quotes in query string' do
+    TwitterSearchWatcher.new('"hello there"').search_url.should end_with('q="hello+there"')
+  end
+
+  it 'should replace all spaces in query string with + signs' do
+    TwitterSearchWatcher.new('hello there').search_url.should end_with('q=hello+there')
+  end
+
+  it 'should be able to execute search' do
+    watcher = TwitterSearchWatcher.new 'chunky bacon'
+    watcher.should_receive(:open).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response)
+    response = watcher.search!
+    response.should be_a_kind_of(Hash)
+    response['query'].should == 'remitaylor' # make sure it's getting our fake date
+  end
+
+  it 'should be able to request a particular :page (pagination)' do
+    TwitterSearchWatcher.new( 'foo', :page => 2 ).search_url.should include("page=2")
+    TwitterSearchWatcher.new( 'foo', :page => 2 ).search_url.should include("q=foo")
+  end
+
+  it 'should be able to request since a particular twitter ID (:since_id)' do
+    TwitterSearchWatcher.new( 'foo', :since_id => 1234 ).search_url.should include("since_id=1234")
+    TwitterSearchWatcher.new( 'foo', :since_id => 1234 ).search_url.should include("q=foo")
+  end
+
+  it 'should be able to request with a particular :max_id (used with :page)' do
+    TwitterSearchWatcher.new( 'foo', :max_id => 4321 ).search_url.should include("max_id=4321")
+    TwitterSearchWatcher.new( 'foo', :max_id => 4321 ).search_url.should include("q=foo")
+  end
+
+  it 'should be able to request with a particular :rpp (results_per_page)' do
+    TwitterSearchWatcher.new( 'foo', :rpp => 90 ).search_url.should include("rpp=90")
+    TwitterSearchWatcher.new( 'foo', :rpp => 90 ).search_url.should include("q=foo")
+
+    lambda { TwitterSearchWatcher.new( 'foo', :rpp => 101 ) }.should raise_error(/100/)
+  end
+
+  it 'should be able to search!(:rpp => 15) ... inotherwords, should be able to pass additional parameters to search!' do
+    watcher = TwitterSearchWatcher.new 'foo'
+
+    watcher.should_receive(:open).with(/99/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
+    watcher.search! :rpp => 99
+
+    watcher.should_receive(:open).with(/foo/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
+    watcher.search! :rpp => 99
+  end
+
+  it 'should be able to search!(:rpp => nil) to override an existing parameter' do
+    watcher = TwitterSearchWatcher.new 'foo', :rpp => 50
+    watcher.search_url.should include('rpp=50')
+    watcher.search_url.should include('q=foo')
+
+    # double check that normal overrides are working properly
+    watcher.search_url( :rpp => 99 ).should_not include('rpp=50')
+    watcher.search_url( :rpp => 99 ).should include('rpp=99')
+
+    watcher.search_url( :rpp => nil ).should_not include('rpp=50')
+    watcher.search_url( :rpp => nil ).should_not include('rpp')
+  end
+
+  it 'should be able to search_newer!(result_set) to get results newer than a given result set' do
+    watcher = TwitterSearchWatcher.new 'chunky bacon'
+    watcher.should_receive(:open).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response(:max_id => 5555))
+    response = watcher.search!
+
+    watcher.should_receive(:open).with(/since_id=5555/, 'User-Agent' => watcher.user_agent).and_return(fake_response)
+    watcher.search_newer! response
+  end
+
+  it 'should be able to search_more!(result_set) to get paginated results' do
+    watcher = TwitterSearchWatcher.new 'chunky bacon'
+    watcher.should_receive(:open).with(watcher.search_url, 'User-Agent' => watcher.user_agent).and_return(fake_response(:page => 1, :max_id => 444, :next_page => '?page=2&max_id=444&q=chunky+bacon'))
+    response = watcher.search!
+    
+    watcher.should_receive(:open).with(/page=2/, 'User-Agent' => watcher.user_agent).and_return(fake_response(:page => 2, :max_id => 445, :next_page => nil))
+    response2 = watcher.search_more! response
+    watcher.should_receive(:open).with(/max_id=444/, 'User-Agent' => watcher.user_agent).and_return(fake_response(:page => 2, :max_id => 445, :next_page => nil))
+    response2 = watcher.search_more! response
+
+    watcher.should_not_receive(:open)
+    watcher.search_more!( response2 ).should be_nil # because no next_page
+  end
+
+  it 'should be able to watch! on a watcher' do
+    # it's a PITA to test the loop, so i'm not currently testing that check_every and max_pages are actually used correctly
+    watcher = TwitterSearchWatcher.new 'chunky bacon', :check_every => 120, :max_pages => 5
+    watcher.check_every.should == 120
+    watcher.max_pages.should == 5
+    watcher.should respond_to(:watch!)
+  end
+
+  it 'default rpp should be 100 (the max)'
+
+  it 'should have a nice method to call that will make a new watcher and start watching'
+
+  it 'should be able to watch! with pagination (will search! while there is a :next_page to request)'
+
+end

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification 
+name: devfu-twitter-search-watcher
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- remi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-06 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: for watching a particular Twitter search and calling code whenever there are new tweets
+email: remi@remitaylor.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/twitter-search-watcher.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/twitter_search_watcher_spec.rb
+has_rdoc: false
+homepage: http://github.com/devfu/twitter-search-watcher
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: for watching a Twitter search
+test_files: 
+- spec/twitter_search_watcher_spec.rb
+- spec/spec_helper.rb