jsl-osprey 0.0.0

data/README.rdoc

@@ -0,0 +1,16 @@
+= Description
+
+Osprey is a tool for watching and grabbing Twitter tweets.  Currently, it provides an interface over the "Search" API.
+
+It keeps track of tweets that have been seen for a particular term so that you don't have to.
+
+= Usage
+
+  results = Osprey::Search.new('swine flu').fetch
+
+  results.records      # Returns all records found as Osprey::Tweet objects
+  results.new_records  # Returns only the records that you haven't seen in response to the query 'swine flu'
+
+= Author
+
+Justin S. Leitgeb, justin@phq.org

data/Rakefile

@@ -0,0 +1,42 @@
+require 'rubygems'
+require 'spec'
+
+require 'rake'
+require 'spec/rake/spectask'
+require 'rake/rdoctask'
+
+require 'lib/osprey'
+
+desc 'Test the plugin.'
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.spec_opts  = ["--format", "progress", "--colour"]
+  t.libs << 'lib'
+  t.verbose = true
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "osprey"
+    gemspec.summary = "A Twitter API that keeps track of tweets you've seen"
+    gemspec.email = "justin@phq.org"
+    gemspec.homepage = "http://github.com/jsl/osprey"
+    gemspec.description = "Osprey interfaces with the Twitter search API and keeps track of tweets you've seen"
+    gemspec.authors = ["Justin Leitgeb"]
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+desc "Run all the tests"
+task :default => :spec
+
+desc 'Generate documentation'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Osprey'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/lib/core_ext/array.rb

@@ -0,0 +1,8 @@
+class Array
+    
+  # From Rails' ActiveSupport library
+  def extract_options!
+    last.is_a?(::Hash) ? pop : {}
+  end
+
+end

data/lib/core_ext/hash.rb

@@ -0,0 +1,5 @@
+class Hash
+  def reverse_merge(other_hash)
+    other_hash.merge(self)
+  end
+end

data/lib/osprey/backend/dev_null.rb

@@ -0,0 +1,21 @@
+module Osprey
+  module Backend
+    
+    # Class exists for cases where you don't want Feedzirra to remember what 
+    # it has fetched.  If this backend is selected, user needs to pass arguments
+    # in the form of a Feed object to the Reader class to help Feedzirra know when
+    # a page hasn't changed, and which feed entries have been updated.
+    class DevNull
+      
+      def initialize(options = { }) 
+      end
+
+      def get(url)
+      end
+    
+      def set(url, result) 
+      end
+      
+    end
+  end
+end

data/lib/osprey/backend/filesystem.rb

@@ -0,0 +1,32 @@
+require 'md5'
+require 'uri'
+
+module Osprey
+  module Backend
+    class Filesystem
+      
+      DEFAULTS = {
+        :path => File.expand_path(File.join(%w[ ~ / .osprey ]))
+      } unless defined?(DEFAULTS)
+      
+      def initialize(options = { })
+        @options = DEFAULTS.merge(options)
+      end
+      
+      def get(url)
+        f = filename_for(url)
+        Marshal.load(File.read(f)) if File.exist?(f)
+      end
+    
+      def set(url, result)
+        File.open(filename_for(url), 'w') {|f| f.write(Marshal.dump(result)) }
+      end
+      
+      private
+      
+      def filename_for(url)
+        File.join(@options[:path], MD5.hexdigest(URI.parse(url).normalize.to_s))
+      end
+    end
+  end
+end

data/lib/osprey/backend/memcache.rb

@@ -0,0 +1,35 @@
+module Osprey
+  module Backend
+    
+    # Can be used to set up Memcache, or clients able to speak the Memcache protocol such as 
+    # Tokyo Tyrant, as a Feedzirra::Backend.
+    class Memcache
+      DEFAULTS = {
+        :server => 'localhost',
+        :port   => '11211'
+      } unless defined?(DEFAULTS)
+      
+      def initialize(options = { })
+        @options  = DEFAULTS.merge(options)
+        @cache    = MemCache.new([ @options[:server], @options[:port] ].join(':'), :namespace => 'Osprey')
+      end
+      
+      def get(url)
+        res = @cache.get(key_for(url))
+        Marshal.load(res) unless res.nil?
+      end
+    
+      def set(url, result)
+        @cache.set(key_for(url), Marshal.dump(result))
+      end
+      
+      private
+      
+      def key_for(url)
+        MD5.hexdigest(URI.parse(url).normalize.to_s)
+      end
+
+    end
+    
+  end
+end

data/lib/osprey/backend/memory.rb

@@ -0,0 +1,22 @@
+module Osprey
+  module Backend
+    
+    # Memory store uses a ruby Hash to store the results of feed fetches.
+    # It won't persist after the application exits.
+    class Memory
+      
+      def initialize(options = { })
+        @store = { }
+      end
+      
+      def get(url)
+        @store[url]
+      end
+    
+      def set(url, result)
+        @store[url] = result
+      end
+      
+    end
+  end
+end

data/lib/osprey/result_set.rb

@@ -0,0 +1,11 @@
+module Osprey
+  class ResultSet < Array
+    def records
+      self
+    end
+    
+    def new_records
+      self.select{|r| r.new_record?}
+    end
+  end
+end

data/lib/osprey/search.rb

@@ -0,0 +1,73 @@
+module Osprey
+  class Search
+    attr_reader :term
+    
+    DEFAULTS = {
+      :backend => {
+        :klass => Osprey::Backend::Memory
+      }
+    } unless defined?(DEFAULTS)
+    
+    def initialize(term, options = { })
+      @term = term
+      @options = options.reverse_merge(DEFAULTS)
+      @backend = initialize_backend(@options[:backend][:klass])
+    end
+    
+    def fetch            
+      res = Curl::Easy.perform(url)
+      
+      if res.response_code == 200
+        parse_tweets(res.body_str)
+      else
+        $stderr.puts "Received invalid twitter response code of #{res.response_code}."
+      end
+    end
+    
+    private
+    
+    # Initializes the on-disk structure that keeps track of the Twitter ids that 
+    # we've seen.  We use a sql lite data store because it handles concurrency
+    # for us, which is important if there are multiple threads hitting the data
+    # store.
+    def initialize_backend(klass)
+      klass.new
+    end
+
+    # Returns an Array of ids for the Tweets in result
+    def twitter_ids_from_json_result(json)
+      json['results'].map{|r| r['id'].to_i }
+    end
+    
+    def twitter_ids_in(id_list)
+      id_list
+    end
+    
+    def parse_tweets(json_text)
+      json = JSON.parse(json_text)
+
+      res = Osprey::ResultSet.new
+      
+      presults = previous_results
+      
+      json['results'].each do |t|
+        new_record = presults.detect{|rec| rec.twitter_id == t['id'] }.nil?
+        res << Osprey::Tweet.new(t.merge(:new_record? => new_record, :twitter_id => t['id']))
+      end
+
+      @backend.set(url, res)
+      
+      res
+    end
+
+    def previous_results
+      @backend.get(url) || [ ]      
+    end
+
+    # Returns the Twitter search URL for @term
+    def url
+      "http://search.twitter.com/search.json?q=#{CGI.escape(@term)}"
+    end
+    
+  end
+end

data/lib/osprey/tweet.rb

@@ -0,0 +1,4 @@
+module Osprey
+  class Tweet < OpenStruct    
+  end
+end

data/lib/osprey.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'singleton'
+require 'curb'
+require 'CGI'
+require 'json'
+require 'sequel'
+require 'memcache'
+require 'ostruct'
+
+lib_dirs =  [ 'core_ext', File.join('osprey', 'backend'), 'osprey' ].map do |d|
+  File.join(File.dirname(__FILE__), d)
+end
+
+lib_dirs.each do |d|
+  Dir[File.join(d, "*.rb")].each {|file| require file }
+end

data/spec/fixtures/result_set.yml

@@ -0,0 +1,197 @@
+--- !seq:Osprey::ResultSet 
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 14559662
+    :iso_language_code: en
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/189894422/IMG00701_normal.JPG
+    :twitter_id: 1701714354
+    :text: School is shut down! due to swine flu!! thats a good thing right?
+    :created_at: Tue, 05 May 2009 01:28:10 +0000
+    :to_user_id: 
+    :id: 1701714354
+    :from_user: moonbrella
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://orangatame.com/products/twitterberry/">TwitterBerry</a>"
+    :from_user_id: 4513402
+    :iso_language_code: en
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/133607145/djeloowel_normal.jpg
+    :twitter_id: 1701714238
+    :text: Blame it on the flu got u feelin blue, blame it on the swine got u feelin blind, blame it on the mex mex mex mex mex mexicans....
+    :created_at: Tue, 05 May 2009 01:28:07 +0000
+    :to_user_id: 
+    :id: 1701714238
+    :from_user: DJELOO
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 14573136
+    :new_record?: false
+    :profile_image_url: http://static.twitter.com/images/default_profile_normal.png
+    :twitter_id: 1701713717
+    :text: Swine Flu
+    :created_at: Tue, 05 May 2009 01:28:04 +0000
+    :to_user_id: 
+    :id: 1701713717
+    :from_user: friendbartrends
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 15592670
+    :iso_language_code: da
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/197468025/images_normal.jpg
+    :twitter_id: 1701713665
+    :text: Destroying swine flu
+    :created_at: Tue, 05 May 2009 01:28:04 +0000
+    :to_user_id: 
+    :id: 1701713665
+    :from_user: premierozone
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 1983836
+    :iso_language_code: en
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/61959717/DD_Oaxaca_Flower_normal.jpg
+    :twitter_id: 1701712780
+    :text: "@M_tm uh, finally, something more disturbing than swine flu..."
+    :created_at: Tue, 05 May 2009 01:27:58 +0000
+    :to_user_id: 125590
+    :to_user: M_tm
+    :id: 1701712780
+    :from_user: Funchilde
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 15567837
+    :iso_language_code: en
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/197027649/waiting_normal.jpg
+    :twitter_id: 1701712037
+    :text: Susie gave me a Lysol Hand Sanitizer that says "Protects from Swine Flu" (Haha) for the bracelet.
+    :created_at: Tue, 05 May 2009 01:27:53 +0000
+    :to_user_id: 
+    :id: 1701712037
+    :from_user: StayPositive_
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 13123935
+    :iso_language_code: eo
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/151854910/20070408041551_nivertia06churras07-01069_normal.jpg
+    :twitter_id: 1701711989
+    :text: "@danilogentili Swine Flu !!!!!"
+    :created_at: Tue, 05 May 2009 01:27:53 +0000
+    :to_user_id: 168227
+    :to_user: danilogentili
+    :id: 1701711989
+    :from_user: pannain
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 1580999
+    :iso_language_code: en
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/59851479/n31700687_7014_normal.jpg
+    :twitter_id: 1701711830
+    :text: realized that after swine flu we'll get Bear pox, followed by beaver fever, and ending with Mongoose Malaria.... Be on your toes!
+    :created_at: Tue, 05 May 2009 01:27:52 +0000
+    :to_user_id: 
+    :id: 1701711830
+    :from_user: BubbaStoned85
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 166576
+    :iso_language_code: en
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/88556721/profpic2_lg_normal.png
+    :twitter_id: 1701711644
+    :text: + "90 people get the swine flu and everyone wants to wear a face mask. Millions get AIDS and nobody wears a condom. Go figure."
+    :created_at: Tue, 05 May 2009 01:27:51 +0000
+    :to_user_id: 
+    :id: 1701711644
+    :from_user: gemmerzz
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitterfeed.com">twitterfeed</a>"
+    :from_user_id: 14562545
+    :iso_language_code: en
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/189807824/1_normal.jpg
+    :twitter_id: 1701711527
+    :text: "#swineflu 6 More Swine Flu Cases Confirmed on LI - LongIslandPress.com http://ad.vu/bir4"
+    :created_at: Tue, 05 May 2009 01:27:50 +0000
+    :to_user_id: 
+    :id: 1701711527
+    :from_user: News_SwineFlu
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 8288437
+    :iso_language_code: en
+    :new_record?: false
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/193624367/100_1084_normal.jpg
+    :twitter_id: 1701710705
+    :text: That darn Swine Flu is getting a tad close for comfort! A school about twenty minutes from mine closed down for a few days tonight!
+    :created_at: Tue, 05 May 2009 01:27:45 +0000
+    :to_user_id: 
+    :id: 1701710705
+    :from_user: tylergraves2009
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 3617869
+    :iso_language_code: en
+    :new_record?: true
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/71116723/jojobo_normal.jpg
+    :twitter_id: 1701710268
+    :text: "Explaining to the children why they don't need to be terrified of swine flu, but why they really do need to wash their hands. #swineflu"
+    :created_at: Tue, 05 May 2009 01:27:42 +0000
+    :to_user_id: 
+    :id: 1701710268
+    :from_user: jojobo
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitter.com/">web</a>"
+    :from_user_id: 3362361
+    :iso_language_code: en
+    :new_record?: true
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/76497810/picture_normal.jpg
+    :twitter_id: 1701709964
+    :text: Another day locked up with Swine Flu outside. Just went to HomeDepot and more then 50 percent of clients have masks there.
+    :created_at: Tue, 05 May 2009 01:27:40 +0000
+    :to_user_id: 
+    :id: 1701709964
+    :from_user: dennisvdheijden
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://twitterfox.net/">TwitterFox</a>"
+    :from_user_id: 643746
+    :iso_language_code: nl
+    :new_record?: true
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/79399738/cami1_normal.jpg
+    :twitter_id: 1701709832
+    :text: "RT: @KYLking: Byron Center High School students celebrate 'swine break' after flu closes school for week http://tinyurl.com/d2b3yh"
+    :created_at: Tue, 05 May 2009 01:27:39 +0000
+    :to_user_id: 
+    :id: 1701709832
+    :from_user: creister
+- !ruby/object:Osprey::Tweet 
+  table: 
+    :source: "<a href="http://iconfactory.com/software/twitterrific">twitterrific</a>"
+    :from_user_id: 7400045
+    :iso_language_code: en
+    :new_record?: true
+    :profile_image_url: http://s3.amazonaws.com/twitter_production/profile_images/150699579/for_my_baby_normal.jpg
+    :twitter_id: 1701709797
+    :text: I know I said I was going to sleep but who sleeps when you can twitter Itslike an epidemic this twitter stuff it spread like the swine flu
+    :created_at: Tue, 05 May 2009 01:27:39 +0000
+    :to_user_id: 
+    :id: 1701709797
+    :from_user: chefjay1121

data/spec/fixtures/swine_flu1.json

@@ -0,0 +1 @@
+{"results":[{"text":"RT @nicfish: Tracking Swine Flu with Microsoft Virtual Earth and Live Search  http:\/\/bit.ly\/YqdDC  --> sweet!","to_user_id":null,"from_user":"tameraclark","id":1701715308,"from_user_id":386286,"iso_language_code":"en","source":"<a href="http:\/\/www.tweetdeck.com\/">TweetDeck<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/188156012\/eye2_normal.jpg","created_at":"Tue, 05 May 2009 01:28:14 +0000"},{"text":"@keepsgettinbetr when did i worry about the swine flu","to_user_id":9477722,"to_user":"keepsgettinbetr","from_user":"Bayleeee","id":1701714956,"from_user_id":7978237,"iso_language_code":"en","source":"<a href="http:\/\/twitterfon.net\/">TwitterFon<\/a>","profile_image_url":"http:\/\/static.twitter.com\/images\/default_profile_normal.png","created_at":"Tue, 05 May 2009 01:28:12 +0000"},{"text":"RT @TelegraphNews: Swine flu: Mexico starts to ease virus restrictions http:\/\/tinyurl.com\/db6jff","to_user_id":null,"from_user":"lili_one","id":1701714620,"from_user_id":4481131,"iso_language_code":"en","source":"<a href="http:\/\/www.tweetdeck.com\/">TweetDeck<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/181901124\/31-01-07_1330_normal.jpg","created_at":"Tue, 05 May 2009 01:28:10 +0000"},{"text":"Up to the minute swine flu origins: the visual explanation!\nRT @tweetmeme Coconut Chutney: When pigs fly http:\/\/tinyurl.com\/d8cobj","to_user_id":null,"from_user":"hiddenground","id":1701714439,"from_user_id":4158379,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/66627032\/frau-freide_normal.jpg","created_at":"Tue, 05 May 2009 01:28:09 +0000"},{"text":"School is shut down! due to swine flu!! thats a good thing right?","to_user_id":null,"from_user":"moonbrella","id":1701714354,"from_user_id":14559662,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/189894422\/IMG00701_normal.JPG","created_at":"Tue, 05 May 2009 01:28:10 +0000"},{"text":"Blame it on the flu got u feelin blue, blame it on the swine got u feelin blind, blame it on the mex mex mex mex mex mexicans....","to_user_id":null,"from_user":"DJELOO","id":1701714238,"from_user_id":4513402,"iso_language_code":"en","source":"<a href="http:\/\/orangatame.com\/products\/twitterberry\/">TwitterBerry<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/133607145\/djeloowel_normal.jpg","created_at":"Tue, 05 May 2009 01:28:07 +0000"},{"text":"Swine Flu","to_user_id":null,"from_user":"friendbartrends","id":1701713717,"from_user_id":14573136,"source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/static.twitter.com\/images\/default_profile_normal.png","created_at":"Tue, 05 May 2009 01:28:04 +0000"},{"text":"Destroying swine flu","to_user_id":null,"from_user":"premierozone","id":1701713665,"from_user_id":15592670,"iso_language_code":"da","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/197468025\/images_normal.jpg","created_at":"Tue, 05 May 2009 01:28:04 +0000"},{"text":"@M_tm uh, finally, something more disturbing than swine flu...","to_user_id":125590,"to_user":"M_tm","from_user":"Funchilde","id":1701712780,"from_user_id":1983836,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/61959717\/DD_Oaxaca_Flower_normal.jpg","created_at":"Tue, 05 May 2009 01:27:58 +0000"},{"text":"Susie gave me a Lysol Hand Sanitizer that says "Protects from Swine Flu" (Haha) for the bracelet.","to_user_id":null,"from_user":"StayPositive_","id":1701712037,"from_user_id":15567837,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/197027649\/waiting_normal.jpg","created_at":"Tue, 05 May 2009 01:27:53 +0000"},{"text":"@danilogentili Swine Flu !!!!!","to_user_id":168227,"to_user":"danilogentili","from_user":"pannain","id":1701711989,"from_user_id":13123935,"iso_language_code":"eo","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/151854910\/20070408041551_nivertia06churras07-01069_normal.jpg","created_at":"Tue, 05 May 2009 01:27:53 +0000"},{"text":"realized that after swine flu we'll get Bear pox, followed by beaver fever, and ending with Mongoose Malaria.... Be on your toes!","to_user_id":null,"from_user":"BubbaStoned85","id":1701711830,"from_user_id":1580999,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/59851479\/n31700687_7014_normal.jpg","created_at":"Tue, 05 May 2009 01:27:52 +0000"},{"text":"+ "90 people get the swine flu and everyone wants to wear a face mask. Millions get AIDS and nobody wears a condom. Go figure."","to_user_id":null,"from_user":"gemmerzz","id":1701711644,"from_user_id":166576,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/88556721\/profpic2_lg_normal.png","created_at":"Tue, 05 May 2009 01:27:51 +0000"},{"text":"#swineflu 6 More Swine Flu Cases Confirmed on LI - LongIslandPress.com http:\/\/ad.vu\/bir4","to_user_id":null,"from_user":"News_SwineFlu","id":1701711527,"from_user_id":14562545,"iso_language_code":"en","source":"<a href="http:\/\/twitterfeed.com">twitterfeed<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/189807824\/1_normal.jpg","created_at":"Tue, 05 May 2009 01:27:50 +0000"},{"text":"That darn Swine Flu is getting a tad close for comfort! A school about twenty minutes from mine closed down for a few days tonight!","to_user_id":null,"from_user":"tylergraves2009","id":1701710705,"from_user_id":8288437,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/193624367\/100_1084_normal.jpg","created_at":"Tue, 05 May 2009 01:27:45 +0000"}],"since_id":0,"max_id":1701715308,"refresh_url":"?since_id=1701715308&q=swine+flu","results_per_page":15,"next_page":"?page=2&max_id=1701715308&q=swine+flu","completed_in":0.032499,"page":1,"query":"swine+flu"}

data/spec/fixtures/swine_flu2.json

@@ -0,0 +1 @@
+{"results":[{"text":"School is shut down! due to swine flu!! thats a good thing right?","to_user_id":null,"from_user":"moonbrella","id":1701714354,"from_user_id":14559662,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/189894422\/IMG00701_normal.JPG","created_at":"Tue, 05 May 2009 01:28:10 +0000"},{"text":"Blame it on the flu got u feelin blue, blame it on the swine got u feelin blind, blame it on the mex mex mex mex mex mexicans....","to_user_id":null,"from_user":"DJELOO","id":1701714238,"from_user_id":4513402,"iso_language_code":"en","source":"<a href="http:\/\/orangatame.com\/products\/twitterberry\/">TwitterBerry<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/133607145\/djeloowel_normal.jpg","created_at":"Tue, 05 May 2009 01:28:07 +0000"},{"text":"Swine Flu","to_user_id":null,"from_user":"friendbartrends","id":1701713717,"from_user_id":14573136,"source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/static.twitter.com\/images\/default_profile_normal.png","created_at":"Tue, 05 May 2009 01:28:04 +0000"},{"text":"Destroying swine flu","to_user_id":null,"from_user":"premierozone","id":1701713665,"from_user_id":15592670,"iso_language_code":"da","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/197468025\/images_normal.jpg","created_at":"Tue, 05 May 2009 01:28:04 +0000"},{"text":"@M_tm uh, finally, something more disturbing than swine flu...","to_user_id":125590,"to_user":"M_tm","from_user":"Funchilde","id":1701712780,"from_user_id":1983836,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/61959717\/DD_Oaxaca_Flower_normal.jpg","created_at":"Tue, 05 May 2009 01:27:58 +0000"},{"text":"Susie gave me a Lysol Hand Sanitizer that says "Protects from Swine Flu" (Haha) for the bracelet.","to_user_id":null,"from_user":"StayPositive_","id":1701712037,"from_user_id":15567837,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/197027649\/waiting_normal.jpg","created_at":"Tue, 05 May 2009 01:27:53 +0000"},{"text":"@danilogentili Swine Flu !!!!!","to_user_id":168227,"to_user":"danilogentili","from_user":"pannain","id":1701711989,"from_user_id":13123935,"iso_language_code":"eo","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/151854910\/20070408041551_nivertia06churras07-01069_normal.jpg","created_at":"Tue, 05 May 2009 01:27:53 +0000"},{"text":"realized that after swine flu we'll get Bear pox, followed by beaver fever, and ending with Mongoose Malaria.... Be on your toes!","to_user_id":null,"from_user":"BubbaStoned85","id":1701711830,"from_user_id":1580999,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/59851479\/n31700687_7014_normal.jpg","created_at":"Tue, 05 May 2009 01:27:52 +0000"},{"text":"+ "90 people get the swine flu and everyone wants to wear a face mask. Millions get AIDS and nobody wears a condom. Go figure."","to_user_id":null,"from_user":"gemmerzz","id":1701711644,"from_user_id":166576,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/88556721\/profpic2_lg_normal.png","created_at":"Tue, 05 May 2009 01:27:51 +0000"},{"text":"#swineflu 6 More Swine Flu Cases Confirmed on LI - LongIslandPress.com http:\/\/ad.vu\/bir4","to_user_id":null,"from_user":"News_SwineFlu","id":1701711527,"from_user_id":14562545,"iso_language_code":"en","source":"<a href="http:\/\/twitterfeed.com">twitterfeed<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/189807824\/1_normal.jpg","created_at":"Tue, 05 May 2009 01:27:50 +0000"},{"text":"That darn Swine Flu is getting a tad close for comfort! A school about twenty minutes from mine closed down for a few days tonight!","to_user_id":null,"from_user":"tylergraves2009","id":1701710705,"from_user_id":8288437,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/193624367\/100_1084_normal.jpg","created_at":"Tue, 05 May 2009 01:27:45 +0000"},{"text":"Explaining to the children why they don't need to be terrified of swine flu, but why they really do need to wash their hands. #swineflu","to_user_id":null,"from_user":"jojobo","id":1701710268,"from_user_id":3617869,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/71116723\/jojobo_normal.jpg","created_at":"Tue, 05 May 2009 01:27:42 +0000"},{"text":"Another day locked up with Swine Flu outside. Just went to HomeDepot and more then 50 percent of clients have masks there.","to_user_id":null,"from_user":"dennisvdheijden","id":1701709964,"from_user_id":3362361,"iso_language_code":"en","source":"<a href="http:\/\/twitter.com\/">web<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/76497810\/picture_normal.jpg","created_at":"Tue, 05 May 2009 01:27:40 +0000"},{"text":"RT: @KYLking: Byron Center High School students celebrate 'swine break' after flu closes school for week http:\/\/tinyurl.com\/d2b3yh","to_user_id":null,"from_user":"creister","id":1701709832,"from_user_id":643746,"iso_language_code":"nl","source":"<a href="http:\/\/twitterfox.net\/">TwitterFox<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/79399738\/cami1_normal.jpg","created_at":"Tue, 05 May 2009 01:27:39 +0000"},{"text":"I know I said I was going to sleep but who sleeps when you can twitter Itslike an epidemic this twitter stuff it spread like the swine flu","to_user_id":null,"from_user":"chefjay1121","id":1701709797,"from_user_id":7400045,"iso_language_code":"en","source":"<a href="http:\/\/iconfactory.com\/software\/twitterrific">twitterrific<\/a>","profile_image_url":"http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/150699579\/for_my_baby_normal.jpg","created_at":"Tue, 05 May 2009 01:27:39 +0000"}],"since_id":0,"max_id":1701714354,"refresh_url":"?since_id=1701714354&q=swine+flu","results_per_page":15,"next_page":"?page=2&max_id=1701714354&q=swine+flu","completed_in":0.023306,"page":1,"query":"swine+flu"}

data/spec/osprey/backend/dev_null_spec.rb

@@ -0,0 +1,35 @@
+require File.join(File.dirname(__FILE__), %w[.. .. spec_helper])
+
+describe Osprey::Backend::DevNull do
+  before do
+    @backend = Osprey::Backend::DevNull.new
+  end
+  
+  it_should_behave_like "all backends"
+  
+  it "should initialize properly" do
+    Osprey::Backend::DevNull.new
+  end
+  
+  describe "#set" do
+    it "should accept two arguments" do
+      lambda {
+        @backend.set('foo', 'nothing') 
+      }.should_not raise_error
+    end
+  end
+  
+  describe "#get" do
+    it "should accept one argument" do
+      lambda {
+        @backend.get('foo') 
+      }.should_not raise_error
+    end
+    
+    it "should return nil even after something is set" do
+      @backend.set('junk', 'something')
+      @backend.get('junk').should be_nil
+    end
+  end
+  
+end

data/spec/osprey/backend/filesystem_spec.rb

@@ -0,0 +1,23 @@
+require File.join(File.dirname(__FILE__), %w[.. .. spec_helper])
+
+describe Osprey::Backend::Filesystem do
+  before do
+    @backend = Osprey::Backend::Filesystem.new({:path => '/tmp'})
+  end
+  
+  it_should_behave_like "all backends"
+  
+  it "should accept a Hash of options for inialization" do
+    Osprey::Backend::Filesystem.new({:path => '/tmp'})
+  end
+  
+  describe "#filename_for" do
+    it "should default to the users' home path plus the MD5 of the normalized URL" do
+      MD5.expects(:hexdigest).returns('bah')
+      uri = mock('uri')
+      uri.expects(:normalize).returns('foo')
+      URI.expects(:parse).returns(uri)
+      Osprey::Backend::Filesystem.new({:path => '/tmp'}).__send__(:filename_for, 'hey').should == '/tmp/bah'
+    end
+  end
+end

data/spec/osprey/backend/memcache_spec.rb

@@ -0,0 +1,25 @@
+require File.join(File.dirname(__FILE__), %w[.. .. spec_helper])
+
+describe Osprey::Backend::Memcache do
+  before do
+    @backend = Osprey::Backend::Memcache.new({:server => 'localhost', :port => 11211})
+    @cache = mock('cache')
+    @backend.instance_variable_set(:@cache, @cache)
+  end
+  
+  it_should_behave_like "all backends"  
+
+  describe "#get" do
+    it "should call #get on the cache" do
+      @cache.expects(:get)
+      @backend.get('foo')
+    end    
+  end
+  
+  describe "#set" do  
+    it "should call #set on the cache" do
+      @cache.expects(:set)
+      @backend.set('foo', 'hey')
+    end
+  end
+end

data/spec/osprey/backend/memory_spec.rb

@@ -0,0 +1,14 @@
+require File.join(File.dirname(__FILE__), %w[.. .. spec_helper])
+
+describe Osprey::Backend::Memory do
+  before do
+    @backend = Osprey::Backend::Memory.new
+  end
+  
+  it_should_behave_like "all backends"  
+  
+  it "should initialize properly" do
+    Osprey::Backend::Memory.new
+  end
+  
+end

data/spec/osprey/result_set_spec.rb

@@ -0,0 +1,15 @@
+require File.join(File.dirname(__FILE__),  %w[ .. spec_helper ])
+
+describe Osprey::ResultSet do
+  before do
+    @rs = YAML.load_file(File.join(File.dirname(__FILE__), %w[.. fixtures result_set.yml]))
+  end
+  
+  it "should return 4 results for #new_records" do
+    @rs.new_records.size.should == 4
+  end
+  
+  it "should return 15 results for #records" do
+    @rs.records.size.should == 15
+  end
+end

data/spec/osprey/search_spec.rb

@@ -0,0 +1,63 @@
+require File.join(File.dirname(__FILE__),  %w[ .. spec_helper ])
+
+describe Osprey::Search do
+  describe "initialization" do
+    it "should initialize without error" do
+      Osprey::Search.new('bar')
+    end
+    
+    it "should create a new instance of the default backend" do
+      Osprey::Search.new('bar').instance_variable_get(:@backend).should be_a(Osprey::Backend::Memory)
+    end
+  end
+
+  it "should set the attr accessor for the term" do 
+    tr = Osprey::Search.new('foo')
+    tr.term.should == 'foo'
+  end
+
+  describe "#fetch" do
+    before do
+      @tr = Osprey::Search.new('swine flu')
+      @curl = mock('curl')
+      @curl.expects(:response_code).returns(200)
+      @swine_flu_json = File.read(File.join(File.dirname(__FILE__), %w[.. fixtures swine_flu1.json]))
+      @curl.expects(:body_str).returns(@swine_flu_json)
+      Curl::Easy.expects(:perform).with('http://search.twitter.com/search.json?q=swine+flu').returns(@curl)
+      @results = @tr.fetch
+    end
+    
+    describe "on the initial request of a term" do
+      it "should return a Osprey::ResultSet" do
+        @results.should be_a(Osprey::ResultSet)
+      end
+
+      it "should contain 15 results" do
+        @results.size.should == 15
+      end
+
+      it "should contain all Osprey::Tweet items" do
+        @results.each{|t| t.should be_a(Osprey::Tweet) }      
+      end
+
+      it "should have all results respond affirmatively to new_record?" do
+        @results.each{|t| t.new_record?.should be_true}
+      end
+    end
+
+    describe "when there are existing results for a term" do
+      before do
+        @curl.expects(:response_code).returns(200)
+        
+        @swine_flu_json = File.read(File.join(File.dirname(__FILE__), %w[.. fixtures swine_flu2.json]))
+        @curl.expects(:body_str).returns(@swine_flu_json)
+        Curl::Easy.expects(:perform).with('http://search.twitter.com/search.json?q=swine+flu').returns(@curl)        
+        @results = @tr.fetch
+      end
+      
+      it "should have 4 records that are not marked as new" do
+        @results.select{|r| r.new_record? }.size.should == 4
+      end
+    end
+  end
+end

data/spec/osprey_spec.rb

@@ -0,0 +1,7 @@
+require File.join(File.dirname(__FILE__),  'spec_helper')
+
+describe Osprey do
+  it "should have loaded the twitter reader" do
+    Osprey::Search
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'mocha'
+require 'spec'
+
+require File.join(File.dirname(__FILE__), %w[.. lib osprey])
+
+Spec::Runner.configure do |config|
+  config.mock_with(:mocha)
+end
+
+shared_examples_for "all backends" do
+  it "should respond to #get" do
+    @backend.should respond_to(:get)
+  end
+  
+  it "should respond to #set" do
+    @backend.should respond_to(:set)
+  end  
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification 
+name: jsl-osprey
+version: !ruby/object:Gem::Version 
+  version: 0.0.0
+platform: ruby
+authors: 
+- Justin Leitgeb
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-05-04 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Osprey interfaces with the Twitter search API and keeps track of tweets you've seen
+email: justin@phq.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- Rakefile
+- lib/core_ext/array.rb
+- lib/core_ext/hash.rb
+- lib/osprey.rb
+- lib/osprey/backend/dev_null.rb
+- lib/osprey/backend/filesystem.rb
+- lib/osprey/backend/memcache.rb
+- lib/osprey/backend/memory.rb
+- lib/osprey/result_set.rb
+- lib/osprey/tweet.rb
+- lib/osprey/search.rb
+- spec/fixtures/result_set.yml
+- spec/fixtures/swine_flu1.json
+- spec/fixtures/swine_flu2.json
+- spec/osprey/backend/dev_null_spec.rb
+- spec/osprey/backend/filesystem_spec.rb
+- spec/osprey/backend/memcache_spec.rb
+- spec/osprey/backend/memory_spec.rb
+- spec/osprey/result_set_spec.rb
+- spec/osprey/search_spec.rb
+- spec/osprey_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/jsl/osprey
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A Twitter API that keeps track of tweets you've seen
+test_files: 
+- spec/osprey/backend/dev_null_spec.rb
+- spec/osprey/backend/filesystem_spec.rb
+- spec/osprey/backend/memcache_spec.rb
+- spec/osprey/backend/memory_spec.rb
+- spec/osprey/result_set_spec.rb
+- spec/osprey/search_spec.rb
+- spec/osprey_spec.rb
+- spec/spec_helper.rb