tweetable 0.1.10

data/Gemfile

@@ -0,0 +1,8 @@
+source :gemcutter
+gem 'logging'
+gem 'twitter'
+
+
+group :development do
+	gem 'rspec'
+end

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.0.1 2009-09-25
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,33 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
+Rakefile
+lib/tweetable.rb
+lib/tweetable/authorization.rb
+lib/tweetable/collection.rb
+lib/tweetable/link.rb
+lib/tweetable/message.rb
+lib/tweetable/persistable.rb
+lib/tweetable/photo.rb
+lib/tweetable/queue.rb
+lib/tweetable/search.rb
+lib/tweetable/twitter_client.rb
+lib/tweetable/twitter_streaming_client.rb
+lib/tweetable/url.rb
+lib/tweetable/user.rb
+script/console
+script/destroy
+script/generate
+spec/collection_spec.rb
+spec/link_spec.rb
+spec/message_spec.rb
+spec/persistable_spec.rb
+spec/queue_spec.rb
+spec/search_spec.rb
+spec/spec.opts
+spec/spec_helper.rb
+spec/twitter_client_spec.rb
+spec/twitter_streaming_client_spec.rb
+spec/user_spec.rb
+tweetable.gemspec

data/PostInstall.txt

@@ -0,0 +1,7 @@
+
+For more information on tweetable, see http://tweetable.rubyforge.org
+
+NOTE: Change this information in PostInstall.txt 
+You can also delete it if you don't want it.
+
+

data/README.rdoc

@@ -0,0 +1,37 @@
+== DESCRIPTION:
+
+== EXAMPLES:
+
+To create a new Tweetable user:
+
+	@user = Tweetable::User.create(:screen_name => 'flippyhead')
+	
+To then grab recent messages, friend counts, and other profile data:
+
+	@user.update_all # will only grab messages since the last known message
+	
+Now you have access to stuff like:
+
+	@user.friend_ids # [34102, 23423, 67567, etc...]
+	@user.friend_ids.size # 102
+	@user.profile_image_url # http://twitter.com/...
+	@user.messages.size  # 202
+	
+Links in messages can be extracted and expanded:
+
+	@message = @user.messages.first
+	@message.parse_links
+
+	@link = @message.links.first # Tweetable::Link
+	@link.url # http://tinyurl.com/yfuhltt
+	@link.long_url # http://pathable.com
+	
+And are connected to other users who mention them:
+	
+	@message.links.size # 2
+	@link.count # 8 (uses discovered so far)
+	@link.users # [<Tweetable::User:0x1 @_attributes={}>, ...]
+	
+Performing a keyword search is just as easy:
+
+	# @search = 

data/Rakefile

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'bundler'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "tweetable"
+    gemspec.summary = "Track twitter messages and users in memory using Redis"
+    gemspec.description = "Track twitter messages and users in memory using Redis"
+    gemspec.email = "peter@flippyhead.com"
+    gemspec.homepage = "http://github.com/flippyhead/tweetable"
+    gemspec.authors = ["Peter T. Brown"]
+    gemspec.add_bundler_dependencies
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.10

data/lib/tweetable/authorization.rb

@@ -0,0 +1,21 @@
+module Tweetable
+  class Authorization < Persistable
+    attribute :oauth_access_token
+    attribute :oauth_access_secret
+    
+    index :oauth_access_token
+    
+    def user_id
+      return if self.oauth_access_token.nil?
+      self.oauth_access_token.split('-')[0] # tokens start with ID as in: 13705052-bz9IrOTwWbLgWHQDKkGnVd815ybTujc0QeXMlh7ZJ
+    end
+    
+    protected
+    
+    def validate
+      assert_present :oauth_access_token
+      assert_present :oauth_access_secret
+      assert_unique :oauth_access_token
+    end
+  end
+end

data/lib/tweetable/collection.rb

@@ -0,0 +1,54 @@
+module Tweetable
+  class Collection < Persistable    
+    attribute :created_at
+    attribute :updated_at    
+    attribute :name  # super class attributes don't get picked up in subclasses for some reason
+    index :name
+        
+    def validate
+      assert_present :name
+      assert_unique :name
+    end
+  end  
+  
+  class UserCollection < Collection
+    attribute :created_at
+    attribute :updated_at
+    attribute :name  
+    index :name
+    
+    set  :user_set, User
+    list :users, User
+  end  
+
+  class MessageCollection < Collection    
+    attribute :created_at
+    attribute :updated_at
+    attribute :name  
+    index :name
+    
+    set  :message_set, Message
+    list :messages, Message
+  end  
+  
+  class SearchCollection < Collection
+    attribute :created_at
+    attribute :updated_at
+    attribute :name  
+    index :name
+    
+    set  :search_set, Search
+    list :searches, Search
+  end    
+  
+  class LinkCollection < Collection
+    attribute :created_at
+    attribute :updated_at
+    attribute :name  
+    index :name
+    
+    set  :link_set, Link
+    list :links, Link
+  end    
+  
+end

data/lib/tweetable/link.rb

@@ -0,0 +1,30 @@
+module Tweetable
+  class Link < Persistable
+    URL_PATTERN = /(http:\S+)/ix
+    
+    attribute :created_at
+    attribute :updated_at
+    attribute :url
+    attribute :long_url  
+    
+    index :url
+    index :long_url
+    
+    # set :messages, Tweetable::Message
+    set :users, Tweetable::User
+    counter :count
+    
+    def increment_usage_count(user)
+      return false if (user.nil? || self.users.include?(user))
+      users.add(user)
+      self.incr(:count)
+    end
+    
+    protected
+    
+    def validate
+      assert_present :url
+      assert_unique :url
+    end
+  end
+end

data/lib/tweetable/message.rb

@@ -0,0 +1,119 @@
+module Tweetable
+  class Message < Persistable    
+    attr_accessor :owner 
+    
+    attribute :created_at
+    attribute :updated_at    
+    attribute :message_id
+    attribute :text
+    attribute :favorited
+    attribute :from_user_id
+    attribute :from_screen_name    
+    attribute :links_parsed
+    attribute :photos_parsed
+    attribute :sent_at
+    attribute :created_at    
+    
+    set :links, Link
+    set :photos, Photo
+    set :tags
+    
+    index :message_id
+    index :links_parsed
+    index :photos_parsed
+    
+    def self.create_from_timeline(message, create_user = false)
+      m = Message.find_or_create(:message_id, message[:id])
+      
+      m.update(
+        :favorited => message.favorited, 
+        :photos_parsed => '0',
+        :links_parsed => '0',
+        :created_at => Time.now.utc.to_s, 
+        :sent_at => message.created_at,
+        :text => message.text, 
+        :from_screen_name => message.user.screen_name.downcase, 
+        :from_user_id => message.user[:id])
+            
+      if create_user and m.valid?
+        u = User.create_from_timeline(message.user)
+        u.messages << m if u.valid?
+      end
+      m
+    end
+    
+    def self.create_from_status(text, client)
+      self.create_from_timeline(client.update(text), true)
+    end
+    
+    def self.purge(&block)
+      all.sort.each do |message|
+        message.delete if yield(message)
+      end
+    end
+    
+    def from_user
+      return nil if self.from_screen_name.nil?
+      User.find(:screen_name => self.from_screen_name.downcase).first
+    end
+    
+    def parse_links(force = false, longify = true)
+      return unless (force or self.links_parsed != '1')
+
+      urls = self.text.scan(Link::URL_PATTERN).flatten
+      urls.each do |url|
+        link = Link.find(:url => url).first
+        
+        if !link
+          link = Link.create(:url => url, :created_at => Time.new.to_s)
+          next if !link.valid?
+          
+          # link.messages.add(self)
+          long_url = URL.lookup_long_url(url)
+          link.update(:long_url => long_url) unless (long_url.nil? or long_url == url)
+        end
+
+        link.increment_usage_count(from_user)
+        
+        update(:links_parsed => '1')
+        links.add(link)
+      end
+      
+      links
+    end
+    
+    def twitter_link
+      "http://twitter.com/#{from_screen_name}/status/#{message_id}"
+    end
+    
+    def validate
+      super
+      assert_unique :message_id
+      assert_present :text
+      assert_format :links_parsed, /^[0,1]$/
+      assert_format :photos_parsed, /^[0,1]$/
+    end
+    
+    def hash
+      self.id.hash
+    end
+
+    # Simply delegate to == in this example.
+    def eql?(comparee)
+      self == comparee
+    end
+
+    # Objects are equal if they have the same
+    # unique custom identifier.
+    def ==(comparee)
+      self.id == comparee.id
+    end
+    
+    # It seems that, at least using streaming, message id's are not sequential anymore 
+    # So comparisons are done on the official sent_at date/time
+    def <=>(o)
+      return 1 if o.nil?
+      Time.parse(self.sent_at) <=> Time.parse(o.sent_at)
+    end
+  end
+end

data/lib/tweetable/persistable.rb

@@ -0,0 +1,32 @@
+require 'ohm'
+
+module Tweetable
+  class Persistable < Ohm::Model
+    attribute :created_at
+    attribute :updated_at
+
+    def self.find_or_create(key, value)      
+      attributes = {key => value} # this persistable uses an old interface
+      models = self.find(attributes)
+      models.empty? ? self.create(attributes.merge(:created_at => Time.now.utc.to_s)) : models.first
+    end
+    
+    def needs_update?(force = false)
+      force or self.updated_at.blank? or (Time.parse(self.updated_at) + self.config[:update_delay]) < Time.now.utc
+    end  
+
+    def client
+      Tweetable.client
+    end
+    
+    def config
+      Tweetable.config
+    end
+    
+    protected
+    
+    def validate
+      
+    end        
+  end
+end

data/lib/tweetable/photo.rb

@@ -0,0 +1,6 @@
+module Tweetable
+  class Photo < Link
+    attribute :created_at
+    attribute :updated_at
+  end
+end

data/lib/tweetable/queue.rb

@@ -0,0 +1,121 @@
+module Tweetable
+  class Queue
+    def self.clear_search_queue(queue_name)
+      collection = Tweetable::SearchCollection.find(:name, queue_name).first
+      return true if collection.nil?
+      collection.searches.size.times{|i| collection.searches.pop}
+      true
+    end
+
+    def self.clear_user_queue(queue_name)
+      collection = Tweetable::UserCollection.find(:name, queue_name).first
+      return true if collection.nil?
+      collection.users.size.times{|i| collection.users.pop}
+      true
+    end
+    
+    def self.add_to_search_queue(queue_name, queries, &block)
+      queue = Tweetable::SearchCollection.find_or_create(:name, queue_name)
+      return unless queue.searches.empty?
+
+      queries.each do |query|
+        search = Tweetable::Search.find_or_create(:query, query)
+
+        yield search if block_given?
+
+        queue.searches << search.id
+      end
+
+      queue
+    end
+
+    def self.add_to_user_queue(queue_name, screen_names, &block)
+      queue = Tweetable::UserCollection.find_or_create(:name, queue_name)
+      return unless queue.users.empty?
+
+      screen_names.each do |screen_name|
+        user = Tweetable::User.find_or_create(:screen_name, screen_name)
+
+        yield user if block_given?        
+
+        queue.users << user.id
+      end
+    end    
+
+    def self.pull_from_search_queue(queue_name)
+      queue = Tweetable::SearchCollection.find(:name, queue_name).first
+      return 0 if (queue.nil? or queue.searches.empty?)
+
+      count = 0
+      while !queue.searches.empty?        
+        search = Tweetable::Search[queue.searches.pop] # have to find object by id in List
+        process_search(search)
+        count += 1
+      end     
+
+      return count
+    end  
+    
+    def self.pull_from_user_queue(queue_name)
+      queue = Tweetable::UserCollection.find(:name, queue_name).first
+      return if (queue.nil? or queue.users.empty?)
+
+      count = 0
+      while !queue.users.empty?        
+        user = Tweetable::User[queue.users.pop] # have to find object by id in List
+        process_user(user)
+        count += 1
+      end     
+
+      return count
+    end
+    
+    def self.process_search(search)
+      pull_from_queue_safely do
+        return search.update_all(true)          
+      end
+    end
+    
+    def self.process_user(user)
+      pull_from_queue_safely do
+        messages = user.update_all(true)
+        user.tags.each do |tag|
+          message_collection = Tweetable::MessageCollection.find_or_create(:name, tag)
+          user_collection = Tweetable::UserCollection.find_or_create(:name, tag)                    
+
+          message_collection.update(:updated_at => Time.now.utc.to_s)
+          user_collection.update(:updated_at => Time.now.utc.to_s)
+
+          user_collection.user_set.add(user)
+          messages.each{|m| message_collection.messages << m.id}
+        end
+        return messages
+      end
+    end    
+    
+    def self.pull_from_queue_safely
+      begin
+        yield
+      rescue Tweetable::TweetableError => e      
+        raise TemporaryPullFromQueueError.new("Twitter unavailable error: #{e}")
+      rescue Twitter::Unavailable => e      
+        raise TemporaryPullFromQueueError.new("Twitter unavailable error: #{e}")      
+      rescue URI::InvalidURIError => e      
+        raise PullFromQueueError.new("Bad uri error: #{e}")
+      rescue ArgumentError => e
+        raise PullFromQueueError.new("Argument error: #{e}")
+      rescue Crack::ParseError => e
+        raise PullFromQueueError.new("Parsing error: #{e}")
+      rescue Twitter::NotFound => e
+        raise PullFromQueueError.new("Account does not exist: #{e}")
+      rescue Twitter::Unauthorized => e
+        raise PullFromQueueError.new("Not authorized error: #{e}")
+      rescue Errno::ETIMEDOUT => e
+        raise PullFromQueueError.new("Connection timed out error: #{e}")
+      rescue Exception => e
+        HoptoadNotifier.notify(e)          
+        raise PullFromQueueError.new("General error: #{e}")
+      end
+    end
+  end  
+end

data/lib/tweetable/search.rb

@@ -0,0 +1,77 @@
+module Tweetable
+  class Search < Persistable
+    SEARCH_PER_PAGE_LIMIT = 100
+    SEARCH_START_PAGE = 1
+  
+    attribute :created_at
+    attribute :updated_at
+    attribute :query
+    index :query 
+    list :messages, Message        
+    
+    def update_all(force = false)
+      return unless needs_update?(force)
+      self.updated_at = Time.now.utc.to_s
+      self.save
+      update_messages
+    end
+    
+    # Perform the search and update messages if any new exist
+    # Do up to 15 requests to collect as many historical messages as possible
+    # Because search API is different than the resto f the Twitter API this needs to be custom (i.e. cannot use xxx_from_timeline methods)
+    def update_messages(pages = 15)
+      most_recent_message = self.messages.first(:order => 'DESC', :by => :message_id) 
+      since_id = most_recent_message.nil? ? 0 : most_recent_message.message_id
+
+      search_messages = []
+      pages.times do |page|
+        s = search(self.query, since_id, SEARCH_PER_PAGE_LIMIT, page+1)
+        break if s.results.nil?
+        search_messages += s.results
+        break if s.results.size < 99                      
+      end
+      
+      search_messages.each do |message|        
+        m = Message.find_or_create(:message_id, message.id)
+        
+        m.update(
+        :message_id => message.id, 
+        :favorited => message.favorited, 
+        :photos_parsed => '0',
+        :links_parsed => '0',          
+        :created_at => Time.now.utc.to_s, 
+        :sent_at => message.created_at,
+        :text => message.text, 
+        :from_screen_name => message.from_user) # we explicitly don't include the user_id provided by search since it's bullshit: http://code.google.com/p/twitter-api/issues/detail?id=214
+          
+        next if !m.valid?
+          
+        # create the user for this message
+        u = User.find_or_create(:screen_name, message.from_user)
+        u.update(
+          :user_id => message.from_user_id,
+          :profile_image_url => message.profile_image_url)
+
+        self.messages << m unless self.messages.include?(m)
+      end
+      
+      search_messages.flatten
+    end
+    
+    
+    private 
+    
+    def validate
+      assert_present :query
+      assert_unique :query
+    end    
+    
+    def search(query, since_id, per_page = SEARCH_PER_PAGE_LIMIT, page = SEARCH_START_PAGE)      
+      begin
+        Twitter::Search.new(query.strip).since(since_id).per_page(per_page).page(page).fetch
+      rescue NoMethodError => e
+        raise TweetableError.new("Temporary problem searching Twitter: #{e}")
+      end
+    end    
+  end
+end

data/lib/tweetable/twitter_client.rb

@@ -0,0 +1,36 @@
+module Tweetable
+  class TwitterClient
+    attr_accessor :consumer_token, :consumer_secret, :authorization #:oauth_access_token, :oauth_access_secret
+    
+    def method_missing(sym, *args, &block)
+      raise TweetableAuthError.new('Not authorized. You must login or authorize the client.') if @client.nil?
+      @client.send sym, *args, &block
+    end      
+    
+    def authorize(token, secret, authorization)
+      raise TweetableAuthError if authorization.nil?
+      
+      self.authorization = authorization
+      self.consumer_token = token
+      self.consumer_secret = secret      
+      self.oauth.authorize_from_access(self.authorization.oauth_access_token, self.authorization.oauth_access_secret)
+            
+      @client = Twitter::Base.new(self.oauth)
+      self
+    end
+    
+    def login(username, password)
+      @client = Twitter::Base.new(Twitter::HTTPAuth.new(username, password))
+      self
+    end
+    
+    def oauth      
+      @oauth ||= Twitter::OAuth.new(self.consumer_token, self.consumer_secret)      
+    end
+    
+    def status
+      status = self.rate_limit_status
+      {:hourly_limit => status[:hourly_limit], :remaining_hits => status[:remaining_hits], :reset_time => status[:reset_time], :reset_time_in_seconds => status[:reset_time_in_seconds]}
+    end  
+  end
+end

data/lib/tweetable/twitter_streaming_client.rb

@@ -0,0 +1,67 @@
+require 'rubygems'
+require 'tweetstream'
+require 'daemons'
+
+module Tweetable
+  class TwitterStreamingClient
+
+    def method_missing(sym, *args, &block)
+      @client.send sym, *args, &block
+    end      
+
+    def initialize(username, password, parser = nil)
+      @client = TweetStream::Client.new(username, password, parser)
+      setup
+      self
+    end
+    
+    def run(query_params)
+      query_params = query_params.call if query_params.kind_of?(Proc)
+      keywords = query_params[:track]
+      keywords = [keywords] unless keywords.kind_of?(Array)
+      
+      Tweetable.log.debug("Tracking keywords: #{query_params.inspect}")
+      
+      self.filter(query_params) do |status|
+        keywords.each do |keyword|
+          store(status, keyword)
+        end
+      end
+    end
+
+    def start(query_params = {}, daemon_options = {}) #:nodoc:
+      Daemons.run_proc('tweetable', daemon_options) do
+        Tweetable.log.debug("Starting...")
+        run(query_params)
+      end
+    end
+
+    def store(status, keyword)
+      if status.text =~ /#{keyword}/i
+        message = Message.create_from_timeline(status, true)            
+        Tweetable.log.debug("[#{keyword}] #{message.sent_at} #{message.text} (#{message.message_id})")
+        
+        search = Search.find_or_create(:query, keyword.downcase)
+        search.update(:updated_at => Time.now.utc.to_s)
+        search.messages << message
+      end
+      
+      message
+    end
+    
+    private 
+    def setup
+      self.on_delete do |status_id, user_id|
+        # do nothing
+      end
+
+      self.on_limit do |skip_count|
+        raise TweetableError.new("Twitter streaming rate limit reached (#{skip_count})")
+      end
+      
+      self.on_error do |message|
+        puts "Error: #{message}"
+      end
+    end    
+  end
+end