pork_sandwich 0.1.0

data/lib/pork/saver.rb

@@ -0,0 +1,211 @@
+# Variations within the saver class:
+# - Type of object being saved (tweets, userinfo, followers, followees)
+# - Which tag(s) to use 
+# - Whether to save, update, or not save 
+# 
+# Similarities within the saver class: 
+# - all recieve an object
+# - all receive a rule set
+module Pork
+  class Saver
+    attr_accessor :rules
+  
+    def initialize(rules = {})
+      @rules = rules
+    end
+
+    def save(to_save, &save_type)
+      save_type.call(to_save, rules)    
+    end
+  end
+end
+TWEET_SAVE = lambda do |tweet_to_save, rules|
+  $LOG.debug "TWEET SAVE"
+  tweet = Tweet.new(:text => tweet_to_save.text, 
+                   :time_of_tweet => tweet_to_save.created_at,
+                   :to_user_id_search => tweet_to_save.to_user_id,
+                   :iso_language_code => tweet_to_save.iso_language_code,
+                   :source => tweet_to_save.source,
+                   :profile_image_url => tweet_to_save.profile_image_url,
+                   :from_user_id_search => tweet_to_save.from_user_id,     
+                   :to_user => tweet_to_save.to_user,
+                   :from_user  => tweet_to_save.from_user,
+                   :status_id  => tweet_to_save.status_id
+                    )
+
+  unless rules['tags'].nil?
+    rules['tags'].each do |tag_name, tag_value|
+      tweet.tag_list << tag_value
+    end
+  end
+
+  tweet.save
+  tweet
+  #If not in DB
+  #Active record save
+  #According to rules 
+  #save associations
+
+end
+
+USER_TWEET_SAVE = lambda do |tweet_to_save, rules|
+  $LOG.debug "USER TWEET SAVE"
+  tweet = Tweet.new(:text => tweet_to_save.text, 
+                   :time_of_tweet => tweet_to_save.created_at,
+                   :to_user_id => tweet_to_save.in_reply_to_user_id,
+                   :source => tweet_to_save.source,
+                   :profile_image_url => tweet_to_save.user.profile_image_url,     
+                   :to_user => tweet_to_save.in_reply_to_screen_name,
+                   :from_user  => tweet_to_save.user.screen_name,
+                   :twitter_account_id => tweet_to_save.user.id,
+                   :status_id  => tweet_to_save.id,
+                   :truncated => tweet_to_save.truncated
+                    )
+
+  unless rules['tags'].nil?
+    rules['tags'].each do |tag_name, tag_value|
+      tweet.tag_list << tag_value
+    end
+  end
+
+  tweet.save
+  tweet
+end
+TWITTER_ACCOUNT_SAVE = lambda do |twitter_account_to_save, rules|
+  $LOG.debug "TWITTER ACCOUNT SAVE"
+  if twitter_account_to_save.class == Pork::TwitterUser
+    twitter_account_attribute_hash = {:twitter_id => twitter_account_to_save.twitter_id,
+                                      :screen_name => twitter_account_to_save.twitter_screen_name}
+    twitter_account = TwitterAccount.new(twitter_account_attribute_hash)
+
+    twitter_account_from_find = TwitterAccount.find(twitter_account.twitter_id) rescue nil
+                                      
+  else
+    twitter_account_attribute_hash = {:twitter_id => twitter_account_to_save.id,
+                        :name => twitter_account_to_save.name, 
+                        :screen_name => twitter_account_to_save.screen_name, 
+                        :description => twitter_account_to_save.description, 
+                        :location => twitter_account_to_save.location, 
+                        :profile_image_url => twitter_account_to_save.profile_image_url, 
+                        :url => twitter_account_to_save.url, 
+                        :protected => twitter_account_to_save.protected, 
+                        :followers_count => twitter_account_to_save.followers_count, 
+                        :statuses_count => twitter_account_to_save.statuses_count, 
+                        :friends_count => twitter_account_to_save.friends_count, 
+                        :profile_background_image_url => twitter_account_to_save.profile_background_image_url, 
+                        :profile_background_tile => twitter_account_to_save.profile_background_tile,
+                        :favourites_count => twitter_account_to_save.favourites_count, 
+                        :time_zone => twitter_account_to_save.time_zone, 
+                        :utc_offset => twitter_account_to_save.utc_offset,
+                        :time_of_user_creation => twitter_account_to_save.created_at,
+                        :profile_background_color => twitter_account_to_save.profile_background_color,
+                        :profile_text_color => twitter_account_to_save.profile_text_color,
+                        :profile_link_color => twitter_account_to_save.profile_link_color,
+                        :profile_sidebar_fill_color => twitter_account_to_save.profile_sidebar_fill_color,
+                        :profile_sidebar_border_color => twitter_account_to_save.profile_sidebar_border_color,
+                        :notifications => twitter_account_to_save.notifications,
+                        :verified => twitter_account_to_save.verified,
+                        :twitter_id_for_search => twitter_account_to_save.twitter_id_for_search}    
+    twitter_account = TwitterAccount.new(twitter_account_attribute_hash)
+
+    twitter_account_from_find = TwitterAccount.find_by_screen_name(twitter_account.screen_name)
+
+  end
+
+
+  if twitter_account_from_find
+
+    twitter_account = twitter_account_from_find
+    unless rules['tags'].nil?
+      rules['tags'].each do |tag_name, tag_value|
+        twitter_account.tag_list << tag_value
+      end
+    end
+    twitter_account.update_attributes(twitter_account_attribute_hash)  
+  else
+    unless rules['tags'].nil?
+      rules['tags'].each do |tag_name, tag_value|
+        twitter_account.tag_list << tag_value
+      end
+    end
+     twitter_account.save
+  end                    
+  
+  twitter_account
+
+end
+CALL_SAVE = lambda do |call_to_save, rules|
+  $LOG.debug "CALL SAVE"
+  call = Call.new(:query => call_to_save.query,
+                  :completed_in =>  call_to_save.completed_in,
+                  :since_id => call_to_save.since_id,
+                  :max_id => call_to_save.max_id,
+                  :refresh_url => call_to_save.refresh_url,
+                  :results_per_page => call_to_save.results_per_page,
+                  :next_page => call_to_save.next_page,
+                  :page => call_to_save.page,
+                  :api => call_to_save.api_id)
+  unless rules['tags'].nil?
+    rules['tags'].each do |tag_name, tag_value|
+      call.tag_list << tag_value
+    end
+  end
+  call.save
+  call
+end
+
+RELATIONSHIP_SAVE = lambda do |users_to_save, rules|
+    $LOG.debug "RELATIONSHIP SAVE"
+  follower = users_to_save[:follower]
+  friend = users_to_save[:friend]      
+
+  twitter_relationship = TwitterRelationship.new(:follower_id => follower.db_object.id, :friend_id => friend.db_object.id, :current => true)
+  if rules[:complete_friend_set]
+    twitter_relationship[:complete_friend_set] = rules[:complete_friend_set]
+  end
+  if rules[:complete_follower_set]
+     twitter_relationship[:complete_follower_set] = rules[:complete_follower_set]
+  end
+
+  unless rules['tags'].nil?
+    rules['tags'].each do |tag_name, tag_value|
+      twitter_relationship.tag_list << tag_value
+    end
+  end
+  twitter_relationship.save
+  twitter_relationship
+end
+
+
+
+REACTION_SAVE = lambda do |reaction_to_save, rules|
+  $LOG.debug "REACTION SAVE"
+  
+  
+  initiator = reaction_to_save[:initiator]
+  responder = reaction_to_save[:responder]
+  tweet = reaction_to_save[:tweet]
+  reaction_type = reaction_to_save[:type]
+
+  tweet_reaction = TweetReaction.new(:tweet_id => tweet.id, 
+                                     :reaction_id => Reaction.find_by_reaction_type(reaction_type).id, 
+                                     :initiator_id => initiator.id, 
+                                     :responder_id => responder.id)
+
+  
+  
+  unless rules['tags'].nil?
+    rules['tags'].each do |tag_name, tag_value|
+      tweet_reaction.tag_list << tag_value
+    end
+  end
+
+  tweet_reaction.save
+  tweet_reaction
+end
+
+TREND_SAVE = lambda do |trend_to_save, rules|
+  trend = Trend.new(:name => trend_to_save[:name], :query => trend_to_save[:query])
+  trend.save
+  trend
+end

data/lib/pork/search.rb

@@ -0,0 +1,69 @@
+module Pork
+  class Search
+    attr_reader :query, :db_ids_created, :desired_count, :from_user, :current_count
+  
+    def initialize(query, options = {})
+      @query = query
+      @desired_count = options[:desired_count] #if nil, will pull as far back as the Search API allows
+      @current_count = 0
+      @from_user = options[:from_user] 
+      @db_ids_created = []
+      @collect_users = options[:collect_users]
+    end
+    
+    def historical_pull
+      @search_params = Twitter::Search.new(@query).per_page(100)
+      @search_params.from(@from_user) if @from_user
+      begin
+        loop do
+          @tweets_pulled = @search_params.dup.fetch.results
+          @tweets_pulled.each do |tweet|
+            tweet.status_id = tweet.id   
+            @db_ids_created << $SAVER.save(tweet, &TWEET_SAVE).id
+            $CRAWLER.append(tweet.from_user) if @collect_users
+            @current_count += 1
+            if reached_desired_count? 
+              break
+            end
+          end
+          if reached_desired_count? or @search_params.query[:max_id] == @tweets_pulled.last.id
+            break
+          else
+            @search_params.query[:max_id] = @tweets_pulled.last.id
+          end
+        end
+      rescue Twitter::Unavailable
+        p "ERROR: Twitter unavailable, trying in 60"
+        sleep 60
+        retry
+      rescue Twitter::NotFound
+        p "ERROR: Info target not found, trying to skip"
+      rescue Crack::ParseError
+        p "Error: JSON Parsing error, trying to skip past problem tweet"
+        @search_params.query[:max_id] -= 1000
+      rescue Errno::ETIMEDOUT
+        $LOG.error "ERROR: Puller timed out, retrying in 10"
+        sleep 10
+        retry
+      rescue Twitter::InformTwitter
+        $LOG.error "ERROR: Twitter internal error, retrying in 30"
+        sleep 30
+        retry
+#      rescue NoMethodError
+#        p "Rate limited; holding off for a bit then trying again"
+#        sleep 600
+#        retry
+      end
+      return true
+    end
+    
+    def reached_desired_count?
+      if @desired_count
+        return @current_count >= @desired_count
+      else
+        return false
+      end
+    end
+    
+  end
+end

data/lib/pork/twitter_user.rb

@@ -0,0 +1,83 @@
+module Pork
+  class TwitterUser
+    attr_accessor :twitter_id, :twitter_screen_name, :crawled, :user_info, :db_object, :puller, :desired_follower_count, :desired_friend_count
+    attr_reader :tweet_db_ids, :follower_relationship_db_ids, :friend_relationship_db_ids
+  
+  
+    def initialize(opts)
+      @twitter_id = sanitize_id(opts[:twitter_id])
+      @twitter_screen_name = opts[:twitter_screen_name]
+      @crawled = opts[:crawled]
+      @user_info = opts[:user_info]
+      @db_object = opts[:db_object]
+      @puller = nil
+      @tweet_db_ids = nil
+      @follower_relationship_db_ids = nil
+      @friend_relationship_db_ids = nil
+      @desired_follower_count = opts[:desired_follower_count]
+      @desired_friend_count = opts[:desired_friend_count]
+    end
+
+    def crawled?
+      @crawled
+    end
+    
+    def search
+      twitter_id ? twitter_id : twitter_screen_name    
+    end
+
+    def sanitize_id(id)
+      id.nil? ? nil : id.to_i 
+    end
+    
+    def pull_tweets
+      pull_result = self.puller.pull(self, &TWEETS)
+      @tweet_db_ids = pull_result[:db_ids]
+      return true
+    end
+    
+    def pull_account_info
+      unless @user_info && @db_object
+        self.update_account_info
+      end
+    end
+    
+    def update_account_info
+      pull_result = self.puller.pull(self, &ACCOUNT_INFO)
+      @user_info, @db_object, @twitter_id, @twitter_screen_name = pull_result[:pull_data], pull_result[:db_object], pull_result[:db_object].twitter_id, pull_result[:db_object].screen_name 
+      return true
+    end
+    
+    def pull_followers
+      pull_result = self.puller.pull(self, &FOLLOWERS)
+      @follower_relationship_db_ids = pull_result[:follower_relationship_db_ids]
+    end
+    
+    def pull_friends
+      pull_result = self.puller.pull(self, &FRIENDS)
+      @friend_relationship_db_ids = pull_result[:friend_relationship_db_ids]
+    end
+    
+    def pull_follower_ids
+      pull_result = self.puller.pull(self, &FOLLOWER_IDS)
+      @follower_relationship_db_ids = pull_result[:follower_relationship_db_ids]
+    end
+    
+    def pull_friend_ids
+      pull_result = self.puller.pull(self, &FRIEND_IDS)
+      @friend_relationship_db_ids = pull_result[:friend_relationship_db_ids]
+    end
+    
+    def puller
+      unless @puller
+        @puller = Pork::Puller.new($AUTH)
+      end
+      @puller
+    end
+    
+    
+    
+    
+  
+  end
+end

data/lib/pork.rb

@@ -0,0 +1,28 @@
+require "#{File.dirname(__FILE__)}/pork/twitter_user"
+require "#{File.dirname(__FILE__)}/pork/puller"
+require "#{File.dirname(__FILE__)}/pork/crawler"
+require "#{File.dirname(__FILE__)}/pork/saver"
+require "#{File.dirname(__FILE__)}/pork/reaction_processor"
+require "#{File.dirname(__FILE__)}/pork/auth"
+require "#{File.dirname(__FILE__)}/pork/log"
+require "#{File.dirname(__FILE__)}/pork/search"
+
+require "#{File.dirname(__FILE__)}/pork/table_classes/reaction"
+require "#{File.dirname(__FILE__)}/pork/table_classes/trend"
+require "#{File.dirname(__FILE__)}/pork/table_classes/tweet"
+require "#{File.dirname(__FILE__)}/pork/table_classes/tweet_reaction"
+require "#{File.dirname(__FILE__)}/pork/table_classes/twitter_account"
+require "#{File.dirname(__FILE__)}/pork/table_classes/twitter_relationship"
+
+module Pork 
+  #Object.send :undef_method, :id
+
+  
+  ActiveRecord::Base.send :include, ActiveRecord::Acts::TaggableOn
+  ActiveRecord::Base.send :include, ActiveRecord::Acts::Tagger
+  
+end
+
+
+
+

data/test/auth_test.rb

@@ -0,0 +1,16 @@
+require "#{File.dirname(__FILE__)}/test_helper"
+
+class AuthTest < Test::Unit::TestCase
+  context "An Auth object" do
+    setup do
+      @auth = Pork::Auth.new("username", "password", "user_agent")
+    end
+
+    should "be able to be created" do
+      assert_equal "username", @auth.auth.client.username
+      assert_equal "password", @auth.auth.client.password
+      assert_equal "user_agent", @auth.auth.client.options[:user_agent]
+    end
+  end
+  
+end

data/test/crawler_test.rb

@@ -0,0 +1,52 @@
+# require "#{File.dirname(__FILE__)}/test_helper"
+# 
+# 
+# class CrawlerTest < Test::Unit::TestCase
+#   context "A crawler initialized with a user array" do
+#     setup do
+#       $CRAWLER = Pork::Crawler.new([Pork::TwitterUser.new(:twitter_id => 15019521, :crawled => false)], 2, &FOLLOWERS_CRAWL)
+#       $PULLER = Pork::Puller.new($AUTH)
+#       $TWITERATOR = Pork::Twiterator.new()
+#       $SAVER = Pork::Saver.new
+#       $LOG = Logger.new(STDOUT)
+#       $LOG.level = Logger::INFO
+#     end
+#     # should "convert the array into a hash" do
+#     #   assert_same_elements $CRAWLER.users, {15019521 => 'uncrawled'} 
+#     # end    
+#     # should "be able to append new and not new users" do
+#     #   $CRAWLER.append('user3')
+#     #   $CRAWLER.append(15019521)
+#     #   assert_same_elements $CRAWLER.users , {15019521 => 'uncrawled','user3' => 'uncrawled'}
+#     # end
+#     # should "return the initial users when called to a depth of 0" do
+#     #   $CRAWLER.depth = 0
+#     #   assert_same_elements [15019521], $CRAWLER.crawl()
+#     # end
+#     # should "be able to crawl 2 depths of follower ids" do
+#     #   $CRAWLER.crawl_type = FOLLOWER_IDS_CRAWL
+#     #   assert_same_elements [15019521, 55555, 4444, 333, 22, 1], $CRAWLER.crawl()
+#     # end
+#     # should "be able to crawl 2 depths of friend ids" do
+#     #   $CRAWLER.crawl_type = FRIEND_IDS_CRAWL
+#     #   assert_same_elements  [15019521, 55555, 4444, 333, 22, 1], $CRAWLER.crawl()
+#     # end
+#     should "be able to gather users from a twitter search" do
+#       $CRAWLER.users = []
+#       $CRAWLER.depth = 0
+#       $CRAWLER.crawl_type = SEARCH_CRAWL 
+#       @test_result = $CRAWLER.crawl('test').map do |u| u.twitter_screen_name end 
+#       assert_same_elements ["SamIsMarth", "MarthIsGreat", "EvanIsLucas", "LucasIsCheap"], @test_result
+#     end
+#     # should "be able to crawl 2 depths of followers" do 
+#     #      $CRAWLER.crawl_type = FOLLOWERS_CRAWL 
+#     #      assert_same_elements [55555,4444,333,22,1,15019521], $CRAWLER.crawl()
+#     #    end
+#     #    should "be able to crawl 2 depths of friends" do 
+#     #      $CRAWLER.crawl_type = FRIENDS_CRAWL 
+#     #       assert_same_elements [55555,4444,333,22,1,15019521], $CRAWLER.crawl()
+#     #    end
+#   end
+#   
+#   
+# end

data/test/factories.rb

@@ -0,0 +1,56 @@
+Factory.sequence :status_id do |n|
+  "1432#{n}".to_i
+end
+
+Factory.define :tweet do |t|
+    t.text "Someting to tweet about"
+    t.from_user "aplusk"
+    t.status_id {Factory.next(:status_id)}       
+end
+
+Factory.define :retweet1_tweet, :parent => :tweet do |t|
+    t.text "RT @username blah blah blah"
+end
+
+Factory.define :retweet2_tweet, :parent => :tweet do |t|
+    t.text "Someting to tweet about RT @username blah blah blah"
+end
+
+Factory.define :mention_tweet, :parent => :tweet do |t|
+    t.text "Someting to @username tweet about"  
+end
+
+Factory.define :reply_tweet, :parent => :tweet do |t|
+    t.text "@username Someting to tweet about"    
+end
+
+Factory.define :double_mention_tweet, :parent => :tweet do |t|
+    t.text "Someting @username1 to @username2 tweet about" 
+end
+
+Factory.sequence :screen_name do |n|
+  "User#{n}"
+end
+
+Factory.sequence :twitter_id do |n|
+  "11#{n}".to_i
+end
+
+Factory.define :twitter_account do |tw|
+    tw.screen_name {Factory.next(:screen_name)}
+    tw.twitter_id {Factory.next(:twitter_id)}
+end
+
+
+Factory.define :tweet_reaction do |tr|
+   tr.reaction {Factory.create(:reaction)}
+   tr.tweet {Factory.create(:tweet)}
+   tr.responder {Factory.create(:twitter_account)}
+   tr.initiator {Factory.create(:twitter_account)}
+end
+
+Factory.define :reaction do |r|
+   r.reaction_type "at mention"
+   r.value 0.6
+end
+