jsl-osprey 0.0.8 → 0.0.8.2

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2007 Justin S. Leitgeb
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -9,14 +9,14 @@ by storing the ids of previous tweets in a global pool which can easily be share
   
 = Installation
 
-  sudo gem install osprey
+  sudo gem install jsl-osprey
 
 = Usage
 
 Basic usage of Osprey is simple, and more advanced usage is easily accomodated by the accepted options and the
 ability to write custom backend modules.
  
-In more basic cases, you can use Osprey as follows:
+In more basic cases, you can use Osprey::Search (the main interface to Osprey) as follows:
  
   require 'osprey'
   
@@ -35,6 +35,8 @@ Note that because this is done, in most cases "results" will be the same as "new
 guarantee that results arrive in order, however, Osprey does checking to make sure that new_results really aren't 
 ones that we've seen before.
 
+See the documentation for Osprey::Search for more information on providing options for the backend configuration of Osprey.
+
 = Processing Details
 
 Osprey implements a simple but effective algorithm for keeping track of which tweets are new and which have been seen before.

data/Rakefile

@@ -0,0 +1,42 @@
+require 'rubygems'
+require 'spec'
+
+require 'rake'
+require 'spec/rake/spectask'
+require 'rake/rdoctask'
+
+require 'lib/osprey'
+
+desc 'Test the plugin.'
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.spec_opts  = ["--format", "progress", "--colour"]
+  t.libs << 'lib'
+  t.verbose = true
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "osprey"
+    gemspec.summary = "A Twitter API that keeps track of tweets you've seen"
+    gemspec.email = "justin@phq.org"
+    gemspec.homepage = "http://github.com/jsl/osprey"
+    gemspec.description = "Osprey interfaces with the Twitter search API and keeps track of tweets you've seen"
+    gemspec.authors = ["Justin Leitgeb"]
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+desc "Run all the tests"
+task :default => :spec
+
+desc 'Generate documentation'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Osprey'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/osprey/**/*.rb')
+end
+

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 5
+:major: 0
+:minor: 0

data/lib/core_ext/array.rb

@@ -0,0 +1,8 @@
+class Array
+    
+  # From Rails' ActiveSupport library
+  def extract_options!
+    last.is_a?(::Hash) ? pop : {}
+  end
+
+end

data/lib/core_ext/hash.rb

@@ -0,0 +1,17 @@
+class Hash
+  # Returns a Hash containing only input keys.
+  # Method from merb-core.
+  def except(*rejected)
+    reject { |k,v| rejected.include?(k) }
+  end
+  
+  def reverse_merge(other_hash)
+    other_hash.merge(self)
+  end
+
+  # Returns a new hash containing only the input keys.
+  # Method from merb-core.
+  def only(*allowed)
+    reject { |k,v| !allowed.include?(k) }
+  end
+end

data/lib/osprey/result_set.rb

@@ -0,0 +1,22 @@
+module Osprey
+  
+  # Container for sets of tweets.
+  class ResultSet < Array
+    attr_accessor :etag
+    
+    # Returns all records.  In most cases, this will be the same as the results
+    # in #new_records, since we add the id of the last tweet fetched to the query 
+    # string automatically.  If Twitter somehow sends us a Tweet that we saw in a
+    # previous fetch, however, this result will return all records indiscriminately
+    # while the method #new_records will be populated with tweets that are really
+    # ones that we haven't seen before.
+    def records
+      self
+    end
+    
+    # Returns only records that are new in the most recent fetch.
+    def new_records
+      self.select{|r| r.new_record == true}
+    end
+  end
+end

data/lib/osprey/search.rb

@@ -0,0 +1,177 @@
+module Osprey
+  
+  # Primary interface to the Osprey Twitter search library.
+  class Search
+    attr_reader :term
+    
+    DEFAULTS = {
+      :backend => {
+        :moneta_klass   => 'Moneta::Memory',
+      },
+      :rpp => 50,
+      :preserved_tweet_ids => 10_000
+    } unless defined?(DEFAULTS)
+
+    # Initializes the Osprey::Search client.
+    #
+    # Usage (Basic):
+    # 
+    #   search = Osprey::Search.new(term, options)
+    #   results = search.fetch
+    #
+    # Usage (Custom Backend):
+    #
+    #   o = Osprey::Search.new('Swine Flu', { :backend => { :moneta_klass => 'Moneta::Memcache', :server => 'localhost:1978' } })
+    #   results = o.fetch
+    # 
+    # Options:
+    #
+    # The Osprey::Search library supports options for customizing the parameters that will be passed to Twitter as well
+    # as to the local backend for keeping track of state.
+    # 
+    # - _backend_ - A hash of options which are passed directly to the Moneta class used as key-value store.  The 
+    #   value for this option should be another Hash containing configuration options for the backend.  Keys supported
+    #   by the backend hash:
+    # 
+    #   - _moneta_backend_ - The backend used for storing the serialized representation of the last fetch of
+    #     this query.  Uses Moneta, a unified interface to key-value storage systems.  This value may be a String, 
+    #     in which case the appropriate Moneta library will be automatically required, or a class constant.  If a 
+    #     class constant is given, it is up to the calling user to require the appropriate Moneta library before referring
+    #     to the constant in the initialization of Osprey::Search.
+    #
+    #   - _other_options_  - Any other options given to the backend hash will be passed directly to the Moneta class 
+    #     used as key-value store for configuration.  Please see the documentation for the appropriate Moneta class for
+    #     options supported.
+    # 
+    # - _preserved_tweet_ids_ - The number of Tweet ids that will be preserved.  This ensures that we're able to detect if
+    #   running different queries returns the same tweet in order to not mark that tweet as a new
+    #   record.  Choosing a higher number here will mean some additional processing time for each
+    #   new tweet, and a small amount of increased storage.  The default should be a decent compromise
+    #   between performance needs while still not seeing duplicate tweets show up as new records.
+    #
+    # - _rpp_ - Requests per page. Determines the results to fetch from Twitter.  Defaults to 50.
+    #
+    # - _since_id_ - Tells twitter to only give us results with an ID greater than the given ID.  Supplied by
+    #   default in URL string if previous results were found.    
+    def initialize(term, options = { })
+      @term    = term
+      @options = options.reverse_merge(DEFAULTS)
+      @backend = initialize_backend(@options[:backend][:moneta_klass], @options[:backend].except(:moneta_klass))
+    end
+    
+    # Returns a Osprey::ResultSet object containing Osprey::Tweet objects for the tweets found for query.
+    def fetch
+      p_results = previous_results
+      
+      res = Curl::Easy.perform(url(p_results))
+            
+      if res.response_code == 200
+        parse_tweets(res, p_results)
+      else
+        $stderr.puts "Received invalid twitter response code of #{res.response_code}."
+      end
+    end
+    
+    private
+    
+    # Initializes the structure that keeps track of the Twitter ids that we've seen.
+    def initialize_backend(klass, options)
+      HashBack::Backend.new('Osprey', klass, options)
+    end
+
+    # Returns an Array of ids for the Tweets in result
+    def twitter_ids_from_json_result(json)
+      json['results'].map{|r| r['id'].to_i }
+    end
+    
+    def twitter_ids_in(id_list)
+      id_list
+    end
+
+    # Determines the etag from the request headers.  Method from Paul Dix's Feedzirra.
+    # 
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A string of the etag or nil if it cannot be found in the headers.
+    def etag_from_header(header)
+      header =~ /.*ETag:\s(.*)\r/
+      $1
+    end
+    
+    def parse_tweets(curl, p_results)
+      json = JSON.parse(curl.body_str)
+      
+      res      = Osprey::ResultSet.new
+      id_pool  = twitter_id_pool
+      
+      json['results'].each do |t|
+        new_record = p_results.detect{|rec| rec.twitter_id == t['id'] }.nil? && !id_pool.include?(t['id'])
+        res << Osprey::Tweet.new(t.merge(:new_record => new_record, :twitter_id => t['id']))
+      end
+
+      add_to_twitter_id_pool(res.new_records)
+      
+      # Store the results from this fetch, unless this set is empty and the previous set is not empty.      
+      if res.empty? && !p_results.nil?
+        res = p_results
+        res.each{|r| r.new_record = false}
+      end
+      
+      @backend[url_key] = res
+      
+      res
+    end
+
+    # Returns an Array of recently-fetched Twitter IDS, or an empty Array
+    # if the backend doesn't have any previous results.
+    def twitter_id_pool
+      # This is a hack to deal with the fact that the backend seems to return an empty Hash when 
+      # a Moneta::Memory store has an uninitialized value.  TODO - take a closer look at this later
+      # and see if Moneta needs to be patched or if it's something screwy in the way we're using it.
+      res = @backend[twitter_id_pool_key]
+      res.nil? || res == { } ? [ ] : res
+    end
+
+    # Adds the given new tweet ids to the twitter id pool so that subsequent requests,
+    # even if they're for other terms, won't see these tweets as new.
+    def add_to_twitter_id_pool(tweets)
+      pool = twitter_id_pool
+      @backend[twitter_id_pool_key] = 
+        ( pool | tweets.map{|t| t.twitter_id} ).sort.reverse.uniq[1..@options[:preserved_tweet_ids]]
+    end
+
+    def twitter_id_pool_key
+      %w[twitter id pool].join('-')
+    end
+
+    # Returns an Array of results from the previous fetch, or an empty
+    # Array if no previous results are detected.
+    def previous_results
+      @backend[url_key] || [ ]
+    end
+
+    # Returns the backend string that will be used for the storage of this term.
+    # Effectively namespaces the urls so that we can use the backend modules for 
+    # storage of other things like a pool of recently-read twitter ids.
+    def url_key
+      term_hash = MD5.hexdigest(@term)
+      ['term', term_hash].join('-')
+    end
+
+    # Returns the Twitter search URL for @term.  Merges the last response ID into the query string
+    # if we have previous results for this query.  This eliminates most duplication from previous 
+    # results and decreases transfer size.
+    def url(previous_response)
+      url = "http://search.twitter.com/search.json?rpp=#{@options[:rpp]}&q=#{CGI.escape(@term)}"
+
+      unless previous_response.nil? || previous_response.empty?
+        last_response_id = previous_response.records.map{|r| r.twitter_id}.sort.last
+        url += "&since_id=#{last_response_id}"
+      end
+
+      url
+    end
+    
+  end
+end

data/lib/osprey/tweet.rb

@@ -0,0 +1,7 @@
+module Osprey
+  
+  # Basic container for a Tweet result.
+  class Tweet < OpenStruct    
+  end
+
+end

data/lib/osprey.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'singleton'
+require 'curb'
+require 'hashback'
+require 'json'
+require 'md5'
+require 'memcache'
+require 'ostruct'
+require 'cgi'
+
+lib_dirs =  [ 'core_ext', 'osprey' ].map do |d|
+  File.join(File.dirname(__FILE__), d)
+end
+
+lib_dirs.each do |d|
+  Dir[File.join(d, "*.rb")].each {|file| require file }
+end

data/osprey.gemspec

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{osprey}
+  s.version = "0.0.8.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Justin Leitgeb"]
+  s.date = %q{2009-05-05}
+  s.description = %q{Osprey interfaces with the Twitter search API and keeps track of tweets you've seen}
+  s.email = %q{justin@phq.org}
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  
+  s.files = ["lib/core_ext/array.rb", "lib/core_ext/hash.rb", "lib/osprey/result_set.rb", 
+    "lib/osprey/search.rb", "lib/osprey/tweet.rb", "lib/osprey.rb", "LICENSE", 
+    "osprey.gemspec", "Rakefile", "README.rdoc", "spec/fixtures/swine_flu1.json", "spec/fixtures/swine_flu2.json", 
+    "spec/fixtures/swine_flu_result_set1.yml", "spec/fixtures/swine_flu_result_set2.yml", 
+    "spec/osprey/result_set_spec.rb", "spec/osprey/search_spec.rb", "spec/osprey_spec.rb", 
+    "spec/spec_helper.rb", "VERSION.yml"]
+  
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/jsl/osprey}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.1}
+  s.summary = %q{A Twitter API that keeps track of tweets you've seen}
+  s.test_files = ["spec/fixtures/swine_flu1.json", "spec/fixtures/swine_flu2.json", 
+    "spec/fixtures/swine_flu_result_set1.yml", "spec/fixtures/swine_flu_result_set2.yml", 
+    "spec/osprey/result_set_spec.rb", "spec/osprey/search_spec.rb", "spec/osprey_spec.rb", "spec/spec_helper.rb"]
+
+  s.extra_rdoc_files = [ "README.rdoc" ]
+  
+  s.rdoc_options += [
+    '--title', 'Osprey',
+    '--main', 'README.rdoc',
+    '--line-numbers',
+    '--inline-source'
+   ]
+
+  %w[ curb wycats-moneta hashback json memcache-client ].each do |dep|
+    s.add_dependency(dep)
+  end
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+  end
+end

data/spec/fixtures/swine_flu1.json

@@ -0,0 +1 @@
+{"results":[{"text":"RT @nicfish: Tracking Swine Flu with Microsoft Virtual Earth and Live Search  http:\/\/bit.ly\/YqdDC  --> sweet!","to_user_id":null,"from_user":"tameraclark","id":1701715308,"from_user_id":386286,"iso_language_code":"en","source":"<a href="http:\/\/www.tweetdeck.com\/">TweetDeck<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/188156012\/eye2_normal.jpg","created_at":"Tue, 05 May 2009 01:28:14 +0000"},{"text":"@keepsgettinbetr when did i worry about the swine flu","to_user_id":9477722,"to_user":"keepsgettinbetr","from_user":"Bayleeee","id":1701714956,"from_user_id":7978237,"iso_language_code":"en","source":"<a href="http:\/\/twitterfon.net\/">TwitterFon<\/a>","profile_image_url":"http:\/\/static.twitter.com\/images\/default_profile_normal.png","created_at":"Tue, 05 May 2009 01:28:12 +0000"},{"text":"RT @TelegraphNews: Swine flu: Mexico starts to ease virus restrictions http:\/\/tinyurl.com\/db6jff","to_user_id":null,"from_user":"lili_one","id":1701714620,"from_user_id":4481131,"iso_language_code":"en","source":"<a href="http:\/\/www.tweetdeck.com\/">TweetDeck<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/181901124\/31-01-07_1330_normal.jpg","created_at":"Tue, 05 May 2009 01:28:10 +0000"},{"text":"Up to the minute swine flu origins: the visual explanation!\nRT @tweetmeme Coconut Chutney: When pigs fly http:\/\/tinyurl.com\/d8cobj","to_user_id":null,"from_user":"hiddenground","id":1701714439,"from_user_id":4158379,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/66627032\/frau-freide_normal.jpg","created_at":"Tue, 05 May 2009 01:28:09 +0000"},{"text":"School is shut down! due to swine flu!! thats a good thing right?","to_user_id":null,"from_user":"moonbrella","id":1701714354,"from_user_id":14559662,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/189894422\/IMG00701_normal.JPG","created_at":"Tue, 05 May 2009 01:28:10 +0000"},{"text":"Blame it on the flu got u feelin blue, blame it on the swine got u feelin blind, blame it on the mex mex mex mex mex mexicans....","to_user_id":null,"from_user":"DJELOO","id":1701714238,"from_user_id":4513402,"iso_language_code":"en","source":"<a href="http:\/\/orangatame.com\/products\/twitterberry\/">TwitterBerry<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/133607145\/djeloowel_normal.jpg","created_at":"Tue, 05 May 2009 01:28:07 +0000"},{"text":"Swine Flu","to_user_id":null,"from_user":"friendbartrends","id":1701713717,"from_user_id":14573136,"source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/static.twitter.com\/images\/default_profile_normal.png","created_at":"Tue, 05 May 2009 01:28:04 +0000"},{"text":"Destroying swine flu","to_user_id":null,"from_user":"premierozone","id":1701713665,"from_user_id":15592670,"iso_language_code":"da","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/197468025\/images_normal.jpg","created_at":"Tue, 05 May 2009 01:28:04 +0000"},{"text":"@M_tm uh, finally, something more disturbing than swine flu...","to_user_id":125590,"to_user":"M_tm","from_user":"Funchilde","id":1701712780,"from_user_id":1983836,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/61959717\/DD_Oaxaca_Flower_normal.jpg","created_at":"Tue, 05 May 2009 01:27:58 +0000"},{"text":"Susie gave me a Lysol Hand Sanitizer that says "Protects from Swine Flu" (Haha) for the bracelet.","to_user_id":null,"from_user":"StayPositive_","id":1701712037,"from_user_id":15567837,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/197027649\/waiting_normal.jpg","created_at":"Tue, 05 May 2009 01:27:53 +0000"},{"text":"@danilogentili Swine Flu !!!!!","to_user_id":168227,"to_user":"danilogentili","from_user":"pannain","id":1701711989,"from_user_id":13123935,"iso_language_code":"eo","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/151854910\/20070408041551_nivertia06churras07-01069_normal.jpg","created_at":"Tue, 05 May 2009 01:27:53 +0000"},{"text":"realized that after swine flu we'll get Bear pox, followed by beaver fever, and ending with Mongoose Malaria.... Be on your toes!","to_user_id":null,"from_user":"BubbaStoned85","id":1701711830,"from_user_id":1580999,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/59851479\/n31700687_7014_normal.jpg","created_at":"Tue, 05 May 2009 01:27:52 +0000"},{"text":"+ "90 people get the swine flu and everyone wants to wear a face mask. Millions get AIDS and nobody wears a condom. Go figure."","to_user_id":null,"from_user":"gemmerzz","id":1701711644,"from_user_id":166576,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/88556721\/profpic2_lg_normal.png","created_at":"Tue, 05 May 2009 01:27:51 +0000"},{"text":"#swineflu 6 More Swine Flu Cases Confirmed on LI - LongIslandPress.com http:\/\/ad.vu\/bir4","to_user_id":null,"from_user":"News_SwineFlu","id":1701711527,"from_user_id":14562545,"iso_language_code":"en","source":"<a href="http:\/\/twitterfeed.com">twitterfeed<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/189807824\/1_normal.jpg","created_at":"Tue, 05 May 2009 01:27:50 +0000"},{"text":"That darn Swine Flu is getting a tad close for comfort! A school about twenty minutes from mine closed down for a few days tonight!","to_user_id":null,"from_user":"tylergraves2009","id":1701710705,"from_user_id":8288437,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/193624367\/100_1084_normal.jpg","created_at":"Tue, 05 May 2009 01:27:45 +0000"}],"since_id":0,"max_id":1701715308,"refresh_url":"?since_id=1701715308&q=swine+flu","results_per_page":15,"next_page":"?page=2&max_id=1701715308&q=swine+flu","completed_in":0.032499,"page":1,"query":"swine+flu"}

data/spec/fixtures/swine_flu2.json

@@ -0,0 +1 @@
+{"results":[{"text":"School is shut down! due to swine flu!! thats a good thing right?","to_user_id":null,"from_user":"moonbrella","id":1701714354,"from_user_id":14559662,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/189894422\/IMG00701_normal.JPG","created_at":"Tue, 05 May 2009 01:28:10 +0000"},{"text":"Blame it on the flu got u feelin blue, blame it on the swine got u feelin blind, blame it on the mex mex mex mex mex mexicans....","to_user_id":null,"from_user":"DJELOO","id":1701714238,"from_user_id":4513402,"iso_language_code":"en","source":"<a href="http:\/\/orangatame.com\/products\/twitterberry\/">TwitterBerry<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/133607145\/djeloowel_normal.jpg","created_at":"Tue, 05 May 2009 01:28:07 +0000"},{"text":"Swine Flu","to_user_id":null,"from_user":"friendbartrends","id":1701713717,"from_user_id":14573136,"source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/static.twitter.com\/images\/default_profile_normal.png","created_at":"Tue, 05 May 2009 01:28:04 +0000"},{"text":"Destroying swine flu","to_user_id":null,"from_user":"premierozone","id":1701713665,"from_user_id":15592670,"iso_language_code":"da","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/197468025\/images_normal.jpg","created_at":"Tue, 05 May 2009 01:28:04 +0000"},{"text":"@M_tm uh, finally, something more disturbing than swine flu...","to_user_id":125590,"to_user":"M_tm","from_user":"Funchilde","id":1701712780,"from_user_id":1983836,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/61959717\/DD_Oaxaca_Flower_normal.jpg","created_at":"Tue, 05 May 2009 01:27:58 +0000"},{"text":"Susie gave me a Lysol Hand Sanitizer that says "Protects from Swine Flu" (Haha) for the bracelet.","to_user_id":null,"from_user":"StayPositive_","id":1701712037,"from_user_id":15567837,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/197027649\/waiting_normal.jpg","created_at":"Tue, 05 May 2009 01:27:53 +0000"},{"text":"@danilogentili Swine Flu !!!!!","to_user_id":168227,"to_user":"danilogentili","from_user":"pannain","id":1701711989,"from_user_id":13123935,"iso_language_code":"eo","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/151854910\/20070408041551_nivertia06churras07-01069_normal.jpg","created_at":"Tue, 05 May 2009 01:27:53 +0000"},{"text":"realized that after swine flu we'll get Bear pox, followed by beaver fever, and ending with Mongoose Malaria.... Be on your toes!","to_user_id":null,"from_user":"BubbaStoned85","id":1701711830,"from_user_id":1580999,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/59851479\/n31700687_7014_normal.jpg","created_at":"Tue, 05 May 2009 01:27:52 +0000"},{"text":"+ "90 people get the swine flu and everyone wants to wear a face mask. Millions get AIDS and nobody wears a condom. Go figure."","to_user_id":null,"from_user":"gemmerzz","id":1701711644,"from_user_id":166576,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/88556721\/profpic2_lg_normal.png","created_at":"Tue, 05 May 2009 01:27:51 +0000"},{"text":"#swineflu 6 More Swine Flu Cases Confirmed on LI - LongIslandPress.com http:\/\/ad.vu\/bir4","to_user_id":null,"from_user":"News_SwineFlu","id":1701711527,"from_user_id":14562545,"iso_language_code":"en","source":"<a href="http:\/\/twitterfeed.com">twitterfeed<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/189807824\/1_normal.jpg","created_at":"Tue, 05 May 2009 01:27:50 +0000"},{"text":"That darn Swine Flu is getting a tad close for comfort! A school about twenty minutes from mine closed down for a few days tonight!","to_user_id":null,"from_user":"tylergraves2009","id":1701710705,"from_user_id":8288437,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/193624367\/100_1084_normal.jpg","created_at":"Tue, 05 May 2009 01:27:45 +0000"},{"text":"Explaining to the children why they don't need to be terrified of swine flu, but why they really do need to wash their hands. #swineflu","to_user_id":null,"from_user":"jojobo","id":1701710268,"from_user_id":3617869,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/71116723\/jojobo_normal.jpg","created_at":"Tue, 05 May 2009 01:27:42 +0000"},{"text":"Another day locked up with Swine Flu outside. Just went to HomeDepot and more then 50 percent of clients have masks there.","to_user_id":null,"from_user":"dennisvdheijden","id":1701709964,"from_user_id":3362361,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/76497810\/picture_normal.jpg","created_at":"Tue, 05 May 2009 01:27:40 +0000"},{"text":"RT: @KYLking: Byron Center High School students celebrate 'swine break' after flu closes school for week http:\/\/tinyurl.com\/d2b3yh","to_user_id":null,"from_user":"creister","id":1701709832,"from_user_id":643746,"iso_language_code":"nl","source":"<a href="http:\/\/twitterfox.net\/">TwitterFox<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/79399738\/cami1_normal.jpg","created_at":"Tue, 05 May 2009 01:27:39 +0000"},{"text":"I know I said I was going to sleep but who sleeps when you can twitter Itslike an epidemic this twitter stuff it spread like the swine flu","to_user_id":null,"from_user":"chefjay1121","id":1701709797,"from_user_id":7400045,"iso_language_code":"en","source":"<a href="http:\/\/iconfactory.com\/software\/twitterrific">twitterrific<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/150699579\/for_my_baby_normal.jpg","created_at":"Tue, 05 May 2009 01:27:39 +0000"}],"since_id":0,"max_id":1701714354,"refresh_url":"?since_id=1701714354&q=swine+flu","results_per_page":15,"next_page":"?page=2&max_id=1701714354&q=swine+flu","completed_in":0.023306,"page":1,"query":"swine+flu"}

data/spec/fixtures/swine_flu_result_set1.yml

@@ -0,0 +1,200 @@
+--- !seq:Osprey::ResultSet 
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/188156012/eye2_normal.jpg
+    :source: "<a href="http://www.tweetdeck.com/">TweetDeck</a>"
+    :created_at: Tue, 05 May 2009 01:28:14 +0000
+    :to_user_id: 
+    :text: "RT @nicfish: Tracking Swine Flu with Microsoft Virtual Earth and Live Search  http://bit.ly/YqdDC  --> sweet!"
+    :new_record?: true
+    :from_user: tameraclark
+    :from_user_id: 386286
+    :twitter_id: 1701715308
+    :iso_language_code: en
+    :id: 1701715308
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://static.twitter.com/images/default_profile_normal.png
+    :source: "<a href="http://twitterfon.net/">TwitterFon</a>"
+    :created_at: Tue, 05 May 2009 01:28:12 +0000
+    :to_user_id: 9477722
+    :text: "@keepsgettinbetr when did i worry about the swine flu"
+    :new_record?: true
+    :from_user: Bayleeee
+    :from_user_id: 7978237
+    :twitter_id: 1701714956
+    :to_user: keepsgettinbetr
+    :iso_language_code: en
+    :id: 1701714956
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/181901124/31-01-07_1330_normal.jpg
+    :source: "<a href="http://www.tweetdeck.com/">TweetDeck</a>"
+    :created_at: Tue, 05 May 2009 01:28:10 +0000
+    :to_user_id: 
+    :text: "RT @TelegraphNews: Swine flu: Mexico starts to ease virus restrictions http://tinyurl.com/db6jff"
+    :new_record?: true
+    :from_user: lili_one
+    :from_user_id: 4481131
+    :twitter_id: 1701714620
+    :iso_language_code: en
+    :id: 1701714620
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/66627032/frau-freide_normal.jpg
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:28:09 +0000
+    :to_user_id: 
+    :text: |-
+      Up to the minute swine flu origins: the visual explanation!
+      RT @tweetmeme Coconut Chutney: When pigs fly http://tinyurl.com/d8cobj
+    :new_record?: true
+    :from_user: hiddenground
+    :from_user_id: 4158379
+    :twitter_id: 1701714439
+    :iso_language_code: en
+    :id: 1701714439
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/189894422/IMG00701_normal.JPG
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:28:10 +0000
+    :to_user_id: 
+    :text: School is shut down! due to swine flu!! thats a good thing right?
+    :new_record?: true
+    :from_user: moonbrella
+    :from_user_id: 14559662
+    :twitter_id: 1701714354
+    :iso_language_code: en
+    :id: 1701714354
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/133607145/djeloowel_normal.jpg
+    :source: "<a href="http://orangatame.com/products/twitterberry/">TwitterBerry</a>"
+    :created_at: Tue, 05 May 2009 01:28:07 +0000
+    :to_user_id: 
+    :text: Blame it on the flu got u feelin blue, blame it on the swine got u feelin blind, blame it on the mex mex mex mex mex mexicans....
+    :new_record?: true
+    :from_user: DJELOO
+    :from_user_id: 4513402
+    :twitter_id: 1701714238
+    :iso_language_code: en
+    :id: 1701714238
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://static.twitter.com/images/default_profile_normal.png
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:28:04 +0000
+    :to_user_id: 
+    :text: Swine Flu
+    :new_record?: true
+    :from_user: friendbartrends
+    :from_user_id: 14573136
+    :twitter_id: 1701713717
+    :id: 1701713717
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/197468025/images_normal.jpg
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:28:04 +0000
+    :to_user_id: 
+    :text: Destroying swine flu
+    :new_record?: true
+    :from_user: premierozone
+    :from_user_id: 15592670
+    :twitter_id: 1701713665
+    :iso_language_code: da
+    :id: 1701713665
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/61959717/DD_Oaxaca_Flower_normal.jpg
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:27:58 +0000
+    :to_user_id: 125590
+    :text: "@M_tm uh, finally, something more disturbing than swine flu..."
+    :new_record?: true
+    :from_user: Funchilde
+    :from_user_id: 1983836
+    :twitter_id: 1701712780
+    :to_user: M_tm
+    :iso_language_code: en
+    :id: 1701712780
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/197027649/waiting_normal.jpg
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:27:53 +0000
+    :to_user_id: 
+    :text: Susie gave me a Lysol Hand Sanitizer that says "Protects from Swine Flu" (Haha) for the bracelet.
+    :new_record?: true
+    :from_user: StayPositive_
+    :from_user_id: 15567837
+    :twitter_id: 1701712037
+    :iso_language_code: en
+    :id: 1701712037
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/151854910/20070408041551_nivertia06churras07-01069_normal.jpg
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:27:53 +0000
+    :to_user_id: 168227
+    :text: "@danilogentili Swine Flu !!!!!"
+    :new_record?: true
+    :from_user: pannain
+    :from_user_id: 13123935
+    :twitter_id: 1701711989
+    :to_user: danilogentili
+    :iso_language_code: eo
+    :id: 1701711989
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/59851479/n31700687_7014_normal.jpg
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:27:52 +0000
+    :to_user_id: 
+    :text: realized that after swine flu we'll get Bear pox, followed by beaver fever, and ending with Mongoose Malaria.... Be on your toes!
+    :new_record?: true
+    :from_user: BubbaStoned85
+    :from_user_id: 1580999
+    :twitter_id: 1701711830
+    :iso_language_code: en
+    :id: 1701711830
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/88556721/profpic2_lg_normal.png
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:27:51 +0000
+    :to_user_id: 
+    :text: + "90 people get the swine flu and everyone wants to wear a face mask. Millions get AIDS and nobody wears a condom. Go figure."
+    :new_record?: true
+    :from_user: gemmerzz
+    :from_user_id: 166576
+    :twitter_id: 1701711644
+    :iso_language_code: en
+    :id: 1701711644
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/189807824/1_normal.jpg
+    :source: "<a href="http://twitterfeed.com">twitterfeed</a>"
+    :created_at: Tue, 05 May 2009 01:27:50 +0000
+    :to_user_id: 
+    :text: "#swineflu 6 More Swine Flu Cases Confirmed on LI - LongIslandPress.com http://ad.vu/bir4"
+    :new_record?: true
+    :from_user: News_SwineFlu
+    :from_user_id: 14562545
+    :twitter_id: 1701711527
+    :iso_language_code: en
+    :id: 1701711527
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/193624367/100_1084_normal.jpg
+    :source: "<a href="http://twitter.com/">web</a>"
+    :created_at: Tue, 05 May 2009 01:27:45 +0000
+    :to_user_id: 
+    :text: That darn Swine Flu is getting a tad close for comfort! A school about twenty minutes from mine closed down for a few days tonight!
+    :new_record?: true
+    :from_user: tylergraves2009
+    :from_user_id: 8288437
+    :twitter_id: 1701710705
+    :iso_language_code: en
+    :id: 1701710705

data/spec/fixtures/swine_flu_result_set2.yml

@@ -0,0 +1,197 @@
+--- !seq:Osprey::ResultSet 
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 14559662
+    :iso_language_code: en
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/189894422/IMG00701_normal.JPG
+    :twitter_id: 1701714354
+    :text: School is shut down! due to swine flu!! thats a good thing right?
+    :created_at: Tue, 05 May 2009 01:28:10 +0000
+    :to_user_id: 
+    :id: 1701714354
+    :from_user: moonbrella
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://orangatame.com/products/twitterberry/">TwitterBerry</a>"
+    :from_user_id: 4513402
+    :iso_language_code: en
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/133607145/djeloowel_normal.jpg
+    :twitter_id: 1701714238
+    :text: Blame it on the flu got u feelin blue, blame it on the swine got u feelin blind, blame it on the mex mex mex mex mex mexicans....
+    :created_at: Tue, 05 May 2009 01:28:07 +0000
+    :to_user_id: 
+    :id: 1701714238
+    :from_user: DJELOO
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 14573136
+    :new_record: false
+    :profile_image_url: http://static.twitter.com/images/default_profile_normal.png
+    :twitter_id: 1701713717
+    :text: Swine Flu
+    :created_at: Tue, 05 May 2009 01:28:04 +0000
+    :to_user_id: 
+    :id: 1701713717
+    :from_user: friendbartrends
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 15592670
+    :iso_language_code: da
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/197468025/images_normal.jpg
+    :twitter_id: 1701713665
+    :text: Destroying swine flu
+    :created_at: Tue, 05 May 2009 01:28:04 +0000
+    :to_user_id: 
+    :id: 1701713665
+    :from_user: premierozone
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 1983836
+    :iso_language_code: en
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/61959717/DD_Oaxaca_Flower_normal.jpg
+    :twitter_id: 1701712780
+    :text: "@M_tm uh, finally, something more disturbing than swine flu..."
+    :created_at: Tue, 05 May 2009 01:27:58 +0000
+    :to_user_id: 125590
+    :to_user: M_tm
+    :id: 1701712780
+    :from_user: Funchilde
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 15567837
+    :iso_language_code: en
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/197027649/waiting_normal.jpg
+    :twitter_id: 1701712037
+    :text: Susie gave me a Lysol Hand Sanitizer that says "Protects from Swine Flu" (Haha) for the bracelet.
+    :created_at: Tue, 05 May 2009 01:27:53 +0000
+    :to_user_id: 
+    :id: 1701712037
+    :from_user: StayPositive_
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 13123935
+    :iso_language_code: eo
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/151854910/20070408041551_nivertia06churras07-01069_normal.jpg
+    :twitter_id: 1701711989
+    :text: "@danilogentili Swine Flu !!!!!"
+    :created_at: Tue, 05 May 2009 01:27:53 +0000
+    :to_user_id: 168227
+    :to_user: danilogentili
+    :id: 1701711989
+    :from_user: pannain
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 1580999
+    :iso_language_code: en
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/59851479/n31700687_7014_normal.jpg
+    :twitter_id: 1701711830
+    :text: realized that after swine flu we'll get Bear pox, followed by beaver fever, and ending with Mongoose Malaria.... Be on your toes!
+    :created_at: Tue, 05 May 2009 01:27:52 +0000
+    :to_user_id: 
+    :id: 1701711830
+    :from_user: BubbaStoned85
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 166576
+    :iso_language_code: en
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/88556721/profpic2_lg_normal.png
+    :twitter_id: 1701711644
+    :text: + "90 people get the swine flu and everyone wants to wear a face mask. Millions get AIDS and nobody wears a condom. Go figure."
+    :created_at: Tue, 05 May 2009 01:27:51 +0000
+    :to_user_id: 
+    :id: 1701711644
+    :from_user: gemmerzz
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitterfeed.com">twitterfeed</a>"
+    :from_user_id: 14562545
+    :iso_language_code: en
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/189807824/1_normal.jpg
+    :twitter_id: 1701711527
+    :text: "#swineflu 6 More Swine Flu Cases Confirmed on LI - LongIslandPress.com http://ad.vu/bir4"
+    :created_at: Tue, 05 May 2009 01:27:50 +0000
+    :to_user_id: 
+    :id: 1701711527
+    :from_user: News_SwineFlu
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 8288437
+    :iso_language_code: en
+    :new_record: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/193624367/100_1084_normal.jpg
+    :twitter_id: 1701710705
+    :text: That darn Swine Flu is getting a tad close for comfort! A school about twenty minutes from mine closed down for a few days tonight!
+    :created_at: Tue, 05 May 2009 01:27:45 +0000
+    :to_user_id: 
+    :id: 1701710705
+    :from_user: tylergraves2009
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 3617869
+    :iso_language_code: en
+    :new_record: true
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/71116723/jojobo_normal.jpg
+    :twitter_id: 1701710268
+    :text: "Explaining to the children why they don't need to be terrified of swine flu, but why they really do need to wash their hands. #swineflu"
+    :created_at: Tue, 05 May 2009 01:27:42 +0000
+    :to_user_id: 
+    :id: 1701710268
+    :from_user: jojobo
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 3362361
+    :iso_language_code: en
+    :new_record: true
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/76497810/picture_normal.jpg
+    :twitter_id: 1701709964
+    :text: Another day locked up with Swine Flu outside. Just went to HomeDepot and more then 50 percent of clients have masks there.
+    :created_at: Tue, 05 May 2009 01:27:40 +0000
+    :to_user_id: 
+    :id: 1701709964
+    :from_user: dennisvdheijden
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitterfox.net/">TwitterFox</a>"
+    :from_user_id: 643746
+    :iso_language_code: nl
+    :new_record: true
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/79399738/cami1_normal.jpg
+    :twitter_id: 1701709832
+    :text: "RT: @KYLking: Byron Center High School students celebrate 'swine break' after flu closes school for week http://tinyurl.com/d2b3yh"
+    :created_at: Tue, 05 May 2009 01:27:39 +0000
+    :to_user_id: 
+    :id: 1701709832
+    :from_user: creister
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://iconfactory.com/software/twitterrific">twitterrific</a>"
+    :from_user_id: 7400045
+    :iso_language_code: en
+    :new_record: true
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/150699579/for_my_baby_normal.jpg
+    :twitter_id: 1701709797
+    :text: I know I said I was going to sleep but who sleeps when you can twitter Itslike an epidemic this twitter stuff it spread like the swine flu
+    :created_at: Tue, 05 May 2009 01:27:39 +0000
+    :to_user_id: 
+    :id: 1701709797
+    :from_user: chefjay1121

data/spec/osprey/result_set_spec.rb

@@ -0,0 +1,15 @@
+require File.join(File.dirname(__FILE__),  %w[ .. spec_helper ])
+
+describe Osprey::ResultSet do
+  before do
+    @rs = YAML.load_file(File.join(File.dirname(__FILE__), %w[.. fixtures swine_flu_result_set2.yml]))
+  end
+  
+  it "should return 4 results for #new_records" do
+    @rs.new_records.size.should == 4
+  end
+  
+  it "should return 15 results for #records" do
+    @rs.records.size.should == 15
+  end
+end

data/spec/osprey/search_spec.rb

@@ -0,0 +1,126 @@
+require File.join(File.dirname(__FILE__),  %w[ .. spec_helper ])
+
+describe Osprey::Search do
+  describe "initialization" do
+    it "should initialize without error" do
+      Osprey::Search.new('bar')
+    end
+    
+    it "should create a new instance of the default backend" do
+      Osprey::Search.new('bar').instance_variable_get(:@backend).should be_a(HashBack::Backend)
+    end
+  end
+
+  it "should set the attr accessor for the term" do 
+    tr = Osprey::Search.new('foo')
+    tr.term.should == 'foo'
+  end
+
+  describe "#fetch" do
+    before do
+      @tr = Osprey::Search.new('swine flu')
+    end
+    
+    describe "when the backend does not contain a previous result set" do
+      before do  
+        @curl = mock('curl')
+        @curl.expects(:response_code).returns(200)
+        @swine_flu_json = File.read(File.join(File.dirname(__FILE__), %w[.. fixtures swine_flu1.json]))
+        @curl.expects(:body_str).returns(@swine_flu_json)
+        Curl::Easy.expects(:perform).with('http://search.twitter.com/search.json?rpp=50&q=swine+flu').returns(@curl)
+        @results = @tr.fetch
+      end
+
+      describe "on the initial request of a term" do
+        it "should return a Osprey::ResultSet" do
+          @results.should be_a(Osprey::ResultSet)
+        end
+
+        it "should contain 15 results" do
+          @results.size.should == 15
+        end
+
+        it "should contain all Osprey::Tweet items" do
+          @results.each{|t| t.should be_a(Osprey::Tweet) }      
+        end
+
+        it "should have all results respond affirmatively to new_record" do
+          @results.each{|t| t.new_record.should be_true}
+        end
+      end
+    end
+    
+    describe "when the backend does contain a previous result set" do
+      before do
+        @curl.expects(:response_code).returns(200)
+        @backend = mock('backend')
+        @backend.expects(:[]).with('twitter-id-pool').twice
+        @backend.expects(:[]).with(@tr.__send__(:url_key)).returns(YAML.load_file(File.join(File.dirname(__FILE__), %w[.. fixtures swine_flu_result_set1.yml])))
+        @backend.expects(:[]=).twice
+        @tr.instance_variable_set(:@backend, @backend)
+        @swine_flu_json = File.read(File.join(File.dirname(__FILE__), %w[.. fixtures swine_flu2.json]))
+        @curl.expects(:body_str).returns(@swine_flu_json)
+        Curl::Easy.expects(:perform).with('http://search.twitter.com/search.json?rpp=50&q=swine+flu&since_id=1701715308').returns(@curl)        
+        @results = @tr.fetch
+      end
+      
+      it "should have 4 records that are not marked as new" do
+        @results.select{|r| r.new_record }.size.should == 4
+      end
+    end
+
+    describe "when feed ids are listed in the pool" do
+      before do
+        @curl.expects(:response_code).returns(200)
+        @backend = mock('backend')
+        @backend.expects(:[]).with('twitter-id-pool').twice.returns([1701715308, 1701714956])
+        @backend.expects(:[]).with(@tr.__send__(:url_key)).returns(nil)
+        @backend.expects(:[]=).twice
+        @tr.instance_variable_set(:@backend, @backend)
+        @swine_flu_json = File.read(File.join(File.dirname(__FILE__), %w[.. fixtures swine_flu1.json]))
+        @curl.expects(:body_str).returns(@swine_flu_json)
+        Curl::Easy.expects(:perform).with('http://search.twitter.com/search.json?rpp=50&q=swine+flu').returns(@curl)        
+        @results = @tr.fetch
+      end
+      
+      it "should return 13 elements in new_records" do
+        @results.new_records.size.should == 13
+      end
+    end
+  end
+  
+  describe "#url_key" do
+    it "should create a key with the MD5 hexdigest of the term" do
+      term = 'swine flu'
+      @tr = Osprey::Search.new(term)
+      @tr.__send__(:url_key).should == "term-#{MD5.hexdigest(term)}"
+    end
+  end
+  
+  describe "#url" do
+    before do
+      @tr = Osprey::Search.new('swine flu')
+      @results = YAML.load_file(File.join(File.dirname(__FILE__), %w[.. fixtures swine_flu_result_set1.yml]))
+    end
+    
+    it "should include the greatest ID in since_if of the given set if results are present" do 
+      url = @tr.__send__(:url, @results)
+      url.should =~ /since_id=1701715308/
+    end
+    
+    it "should not contain since_id if the previous results are nil" do
+      url = @tr.__send__(:url, nil)
+      url.should_not =~ /since_id/      
+    end
+    
+    it "should not contain since_id if the previous results contain no records" do
+      url = @tr.__send__(:url, [])
+      url.should_not =~ /since_id/      
+    end
+    
+    it "should contain an option rpp (results per page) that is set to the @options[:rpp] value" do
+      url = @tr.__send__(:url, [])
+      url.should =~ /rpp=50/      
+    end
+  end
+end

data/spec/osprey_spec.rb

@@ -0,0 +1,7 @@
+require File.join(File.dirname(__FILE__),  'spec_helper')
+
+describe Osprey do
+  it "should have loaded the twitter reader" do
+    Osprey::Search
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'mocha'
+require 'spec'
+
+require File.join(File.dirname(__FILE__), %w[.. lib osprey])
+
+Spec::Runner.configure do |config|
+  config.mock_with(:mocha)
+end
+
+shared_examples_for "all backends" do
+  it "should respond to #get" do
+    @backend.should respond_to(:get)
+  end
+  
+  it "should respond to #set" do
+    @backend.should respond_to(:set)
+  end  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jsl-osprey
 version: !ruby/object:Gem::Version 
-  version: 0.0.8
+  version: 0.0.8.2
 platform: ruby
 authors: 
 - Justin Leitgeb
@@ -71,7 +71,25 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 files: 
+- lib/core_ext/array.rb
+- lib/core_ext/hash.rb
+- lib/osprey/result_set.rb
+- lib/osprey/search.rb
+- lib/osprey/tweet.rb
+- lib/osprey.rb
+- LICENSE
+- osprey.gemspec
+- Rakefile
 - README.rdoc
+- spec/fixtures/swine_flu1.json
+- spec/fixtures/swine_flu2.json
+- spec/fixtures/swine_flu_result_set1.yml
+- spec/fixtures/swine_flu_result_set2.yml
+- spec/osprey/result_set_spec.rb
+- spec/osprey/search_spec.rb
+- spec/osprey_spec.rb
+- spec/spec_helper.rb
+- VERSION.yml
 has_rdoc: true
 homepage: http://github.com/jsl/osprey
 post_install_message: 
@@ -104,5 +122,12 @@ rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
 summary: A Twitter API that keeps track of tweets you've seen
-test_files: []
-
+test_files: 
+- spec/fixtures/swine_flu1.json
+- spec/fixtures/swine_flu2.json
+- spec/fixtures/swine_flu_result_set1.yml
+- spec/fixtures/swine_flu_result_set2.yml
+- spec/osprey/result_set_spec.rb
+- spec/osprey/search_spec.rb
+- spec/osprey_spec.rb
+- spec/spec_helper.rb