pork_sandwich 0.1.0

data/Rakefile

@@ -0,0 +1,79 @@
+require 'rake/testtask'
+require 'active_record'
+require 'rcov'
+require 'metric_fu'
+require 'acts-as-taggable-on'
+require 'twitter'
+
+task :default => :test
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+desc "Migrate the database through scripts in db/migrate. Target specific version with VERSION=x"
+task :migrate => :connect do
+  ActiveRecord::Migrator.migrate('db/migrate', ENV["VERSION"] ? ENV["VERSION"].to_i : nil )
+end
+
+
+desc "Loads the database environment"
+task :environment => :connect do 
+  ActiveRecord::Base.send :include, ActiveRecord::Acts::TaggableOn
+  ActiveRecord::Base.send :include, ActiveRecord::Acts::Tagger
+  
+  require 'lib/pork'
+end
+
+task :connect do
+  ActiveRecord::Base.establish_connection(YAML::load(File.open("config/database.yml"))['production'])
+  ActiveRecord::Base.logger = Logger.new(File.open('database.log', 'a'))
+  
+end
+
+MetricFu::Configuration.run do |config|
+  #define which metrics you want to use
+  #eliminated "stats" because it's specific to rails
+  config.metrics  = [:churn, :saikuro, :flog, :flay, :reek, :roodi, :rcov]
+  config.graphs   = [:flog, :flay, :reek, :roodi, :rcov]
+  config.flay     = { :dirs_to_flay => ['lib']  } 
+  config.flog     = { :dirs_to_flog => ['lib']  }
+  config.reek     = { :dirs_to_reek => ['lib']  }
+  config.roodi    = { :dirs_to_roodi => ['lib'] }
+  config.saikuro  = { :output_directory => 'scratch_directory/saikuro', 
+                      :input_directory => ['lib'],
+                      :cyclo => "",
+                      :filter_cyclo => "0",
+                      :warn_cyclo => "5",
+                      :error_cyclo => "7",
+                      :formater => "text"} 
+  config.churn    = { :start_date => "1 year ago", :minimum_churn_count => 10}
+  config.rcov     = { :test_files => ['test/*_test.rb'],
+                      :rcov_opts => ["--sort coverage", 
+                                     "--no-html", 
+                                     "--text-coverage",
+                                     "--no-color",
+                                     "--profile"]}
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "pork_sandwich"
+    gemspec.summary = "A tool for pulling and storing delicious, delicious Twitter data"
+    gemspec.description = "Ideal for pulling Twitter search tweets, tweets from a twitter account, twitter account info, twitter relationship data, and trends. All data is stored in a handy schema for easy access."
+    gemspec.email = "sam.o.gilbert@gmail.com"
+    gemspec.homepage = "http://github.com/sam1vp/pork_sandwich"
+    gemspec.authors = ["Sam Gilbert", "Evan Burchard"]
+    gemspec.add_dependency('acts-as-taggable-on', '>= 1.0.12')
+    gemspec.add_dependency('twitter', '>= 0.7.9')
+    gemspec.files = FileList['lib/pork/*.rb', 'lib/pork.rb', 'lib/table_classes/*.rb', 'generators/pork_sandwich_migration/*.rb', 'generators/pork_sandwich_migration/templates/*.rb', 'Rakefile', 'README', 'VERSION' ]
+    gemspec.test_files = ['test/*.rb']
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/generators/pork_sandwich_migration/pork_sandwich_migration_generator.rb

@@ -0,0 +1,7 @@
+class PorkSandwichMigrationGenerator < Rails::Generator::Base
+  def manifest
+    record do |m|
+      m.migration_template 'migration.rb', 'db/migrate', :migration_file_name => "pork_sandwich_migration"
+    end
+  end
+end

data/generators/pork_sandwich_migration/templates/pork_sandwich_migration.rb

@@ -0,0 +1,147 @@
+class PorkSandwichMigration < ActiveRecord::Migration
+  self.up
+    create_table :twitter_accounts do |t|
+      t.integer :twitter_id, :limit => 8
+      t.integer :twitter_id_for_search, :limit => 8
+      t.string :screen_name
+      t.string :name 
+      t.string :location
+      t.datetime :time_of_user_creation 
+      t.text :description
+      t.string :profile_image_url
+      t.string :url
+      t.boolean :protected
+      t.integer :followers_count
+      t.string :profile_background_color
+      t.string :profile_text_color
+      t.string :profile_link_color
+      t.string :profile_sidebar_fill_color
+      t.string :profile_sidebar_border_color
+      t.integer :friends_count
+      t.integer :favourites_count
+      t.integer :utc_offset
+      t.string :time_zone
+      t.string :profile_background_image_url
+      t.boolean :profile_background_tile
+      t.integer :statuses_count
+      t.boolean :notifications
+      t.boolean :verified
+      t.timestamps
+    end
+  
+    create_table :tweets do |t|
+      t.string :text, :null => false
+      t.datetime :time_of_tweet 
+      t.string :from_user, :null => false
+      t.integer :to_user_id
+      t.integer :to_user_id_search
+      t.string :to_user      
+      t.integer :status_id, :null => false, :limit => 8       
+      t.integer :from_user_id_search, :limit => 8 
+      t.string :iso_language_code
+      t.string :source
+      t.string :profile_image_url
+      t.integer :twitter_account_id, :limit => 8
+      t.boolean :truncated
+      t.timestamps
+    end
+
+    create_table :trends do  |t|
+      t.string :topic    
+      t.timestamps
+    end
+
+    create_table :tags do  |t|
+      t.column :name, :string
+    end
+  
+    create_table :taggings do  |t|
+      t.column :tag_id, :integer
+      t.column :taggable_id, :integer
+      t.column :tagger_id, :integer
+      t.column :tagger_type, :string
+      t.column :taggable_type, :string
+      t.column :context, :string
+      t.column :created_at, :datetime
+    end
+
+    create_table :twitter_relationships do  |t|
+      t.integer :follower_id, :null => false, :dependent => :destroy, :limit => 8
+      t.integer :friend_id, :null => false, :dependent => :destroy, :limit => 8
+      t.boolean :current
+      t.boolean :complete_follower_set
+      t.boolean :complete_friend_set
+      t.timestamps      
+    end
+
+
+    create_table :trends_tweets, :id => false  do |t|
+      t.integer :tweet_id, :null => false, :dependent => :destroy, :limit => 8
+      t.integer :trend_id, :null => false, :dependent => :destroy, :limit => 8
+    end
+
+    create_table :trends_twitter_accounts, :id => false  do |t|
+      t.integer :trend_id, :null => false, :dependent => :destroy, :limit => 8
+      t.integer :twitter_account_id, :null => false, :dependent => :destroy, :limit => 8
+    end
+
+    create_table :reactions do  |t|
+      t.string :reaction_type, :limit => 20
+      t.float :value
+    end
+
+    create_table :tweet_reactions do |t|
+      t.integer :initiator_id, :null => false, :dependent => :destroy, :limit => 8
+      t.integer :responder_id, :null => false, :dependent => :destroy, :limit => 8
+      t.integer :tweet_id, :null => false, :dependent => :destroy, :limit => 8
+      t.references :reaction
+      t.boolean :current
+      t.timestamps            
+    end
+
+
+    execute "alter table tweets " +  
+    "alter column time_of_tweet type timestamp with time zone;"
+    execute "alter table tweets " +  
+    "alter column created_at type timestamp with time zone;"
+    execute "alter table tweets " +  
+    "alter column updated_at type timestamp with time zone;"  
+    execute "alter table twitter_accounts " +  
+    "alter column time_of_user_creation type timestamp with time zone;"
+    execute "alter table twitter_accounts " +  
+    "alter column created_at type timestamp with time zone;"
+    execute "alter table twitter_accounts " +  
+    "alter column updated_at type timestamp with time zone;"  
+    execute "alter table taggings " +  
+      "alter column created_at type timestamp with time zone;"
+    execute "alter table trends " +  
+    "alter column created_at type timestamp with time zone;"
+    execute "alter table trends " +  
+    "alter column updated_at type timestamp with time zone;"
+    execute "alter table tweet_reactions " +  
+    "alter column created_at type timestamp with time zone;"
+    execute "alter table tweet_reactions " +  
+    "alter column updated_at type timestamp with time zone;"
+    
+  reaction_types = ['retweet', 'mention', 'reply']
+  reaction_types.each do |r|
+    unless Reaction.find_by_reaction_type(r)
+      Reaction.create(:reaction_type => r)
+    end
+  end  
+    
+  end
+
+  def self.down
+    drop_table :twitter_accounts 
+    drop_table :tweets
+    drop_table :trends
+    drop_table :tags
+    drop_table :taggings
+    drop_table :twitter_relationships
+    drop_table :trends_tweets
+    drop_table :trends_twitter_accounts
+    drop_table :reactions
+    drop_table :tweet_reactions
+  end
+end

data/lib/pork/auth.rb

@@ -0,0 +1,11 @@
+module Pork
+  class Auth
+    attr_accessor :auth
+  
+    def initialize(username, password, user_agent = '')
+      @auth = Twitter::Base.new(Twitter::HTTPAuth.new(username, password, :user_agent => user_agent))
+    end
+
+  end
+end
+

data/lib/pork/config.rb

@@ -0,0 +1,3 @@
+module Pork
+  $USER_AGENT = 'web_ecology'
+end

data/lib/pork/crawler.rb

@@ -0,0 +1,91 @@
+module Pork
+  class Crawler
+    attr_accessor :users, :depth, :crawl_type  
+    def initialize(users = [], depth = 0, crawl_type = nil, count = nil)
+      @depth = depth
+      @crawl_type = crawl_type_str_to_proc(crawl_type)
+      @users = users
+      @count = count
+
+    end
+  
+    def crawl_type_str_to_proc(str)
+      proc_dict = {'followers' => FOLLOWERS_CRAWL, 'friends' => FRIENDS_CRAWL, 'follower_ids' => FOLLOWER_IDS_CRAWL, 'friend_ids' => FRIEND_IDS_CRAWL}
+      proc_dict[str]
+    end
+  # crawl type = RT_TO_USER_CRAWL, RT_FROM_USER_CRAWL, 'mention_to_user' 'mention_from_user', 'reply_to_user', 'reply_from_user',
+  
+    def crawl(search_query = nil)
+      unless @users.empty?
+        while @depth > 0
+          @users.dup.each do |user|
+            unless user.crawled?
+              @crawl_type.call(user, search_query, @count)
+              @users.each do |u|
+                if u.search == user
+                u.crawled = true
+              end
+            end
+          end
+        end
+        @depth -= 1
+      end
+      else
+        @crawl_type.call(nil,search_query, @count)
+      end
+      @users  # .collect { |u| u.search }
+    end
+  
+ 
+    def append(user, user_info = nil)
+      user_search_args = @users.collect do |u|
+        u.search
+      end
+      unless user_search_args.include?(user)
+        if user.class == String
+          @user = Pork::TwitterUser.new(:crawled => false, :twitter_screen_name => user)          
+        else
+          @user = Pork::TwitterUser.new(:crawled => false, :twitter_id => user)
+        end
+        if user_info
+          @user.user_info = user_info 
+        end
+ 
+        @users << @user 
+      end
+    end
+  end
+end
+
+SEARCH_CRAWL = lambda do |user, search_query, count|
+  $LOG.info "SEARCH CRAWL"
+  $TWITERATOR.twiterate({:collect_users => true}, {:search_query => search_query}, &SEARCH_ITER)
+  # @users.keys
+  search_query
+end
+
+FOLLOWER_IDS_CRAWL = lambda do |user, search_query, count|
+  $LOG.info "FOLLOWER IDS CRAWL"
+  $PULLER.pull({:user_id => user, :collect_users => true}, &FOLLOWER_IDS_PULL)
+end
+
+FRIEND_IDS_CRAWL = lambda do |user, search_query, count|
+  $LOG.info "FRIEND IDS CRAWL"
+  $PULLER.pull({:user_id => user, :collect_users => true}, &FRIEND_IDS_PULL)
+end
+
+FOLLOWERS_CRAWL = lambda do |user, search_query, count|
+  $LOG.info "FOLLOWERS CRAWL"
+  if not user.db_object
+    user.db_object = $PULLER.pull({:user => user}, &USER_PULL)
+  end
+  $TWITERATOR.twiterate({:count => count}, {:collect_users => true, :user => user}, &FOLLOWERS_ITER)    
+end
+
+FRIENDS_CRAWL = lambda do |user, search_query, count|
+  $LOG.info "FRIENDS CRAWL"
+  if not user.db_object
+    user.db_object = $PULLER.pull({:user => user}, &USER_PULL)
+  end
+  $TWITERATOR.twiterate({ :count => count}, {:collect_users => true, :user => user}, &FRIENDS_ITER)    
+end

data/lib/pork/log.rb

@@ -0,0 +1,16 @@
+module Pork
+  class Log
+    attr_accessor :log, :researcher, :project, :output
+  
+    def initialize(output, options = {})
+      @log = Logger.new(output)
+      @log.level = "info"
+      @researcher = options[:researcher] if options[:researcher]
+      @project = options[:project] if options[:project]
+    end
+  
+    def write(message)
+      @log.info("#{@project}, #{@researcher}, #{Time.now}: " + message)
+    end
+  end
+end

data/lib/pork/puller.rb

@@ -0,0 +1,141 @@
+module Pork
+  class Puller
+    attr_accessor :auth_object
+    
+    def initialize(auth_object = nil)
+      @auth_object = auth_object
+    end
+  
+    def pull(user_object, &pull_type)
+        @user = user_object
+      begin
+        pull_type.call(@user, @auth_object)
+      # rescue Twitter::Unauthorized  
+      rescue Twitter::Unavailable
+        $LOG.error "ERROR: Twitter unavailable, trying in 60"
+        sleep 60
+        retry
+      rescue Twitter::NotFound
+        $LOG.error "ERROR: Info target not found, trying to skip"
+      # rescue Crack::ParseError
+      #   raise Crack::ParseError
+      rescue Errno::ETIMEDOUT
+        $LOG.error "ERROR: Puller timed out, retrying in 10"
+        sleep 10
+        retry
+      rescue Twitter::InformTwitter
+        $LOG.error "ERROR: Twitter internal error, retrying in 30"
+        sleep 30
+        retry
+      end
+    end
+
+  end
+end
+
+
+
+
+ACCOUNT_INFO = lambda do |user, auth_object|
+  $LOG.info "USER PULL"
+  @pull_data = auth_object.user(user.search)  
+  {:pull_data => @pull_data, :db_object => $SAVER.save(@pull_data, &TWITTER_ACCOUNT_SAVE)}
+end
+
+FOLLOWERS = lambda do |user, auth_object|
+  user.pull_account_info
+  rules = {:user_id => user.twitter_id}
+  follower_relationship_db_ids = []
+  # unless user.desired_follower_count
+  #   $SAVER.rules[:complete_follower_set] = true
+  # end
+  loop do
+    rules[:cursor] = -1 if !rules[:cursor]
+    @pull_data = auth_object.followers(rules)
+    @pull_data.users.each do |follower_mash|
+      db_user_object = $SAVER.save(follower_mash, &TWITTER_ACCOUNT_SAVE)  
+      follower_relationship_db_ids << $SAVER.save({:friend => user, :follower => Pork::TwitterUser.new(:twitter_id => follower_mash.id, :twitter_screen_name => follower_mash.screen_name, :db_object => db_user_object)}, &RELATIONSHIP_SAVE).id 
+    end
+    if @pull_data.next_cursor == 0
+      break
+    else
+      rules[:cursor] = @pull_data.next_cursor
+    end
+  end
+  {:follower_relationship_db_ids => follower_relationship_db_ids}
+end
+
+FOLLOWER_IDS = lambda do |user, auth_object|
+  user.pull_account_info
+  # rules[:user_id] = user.twitter_id
+  $SAVER.rules[:complete_follower_set] = true
+  follower_relationship_db_ids = []
+  @pull_data = auth_object.follower_ids({:user_id => user.twitter_id})
+  @pull_data.each do |user_id| 
+    db_user_object = $SAVER.save(Pork::TwitterUser.new(:twitter_id => user_id), &TWITTER_ACCOUNT_SAVE)
+    follower_relationship_db_ids << $SAVER.save({:friend => user, :follower => Pork::TwitterUser.new(:twitter_id => user_id, :db_object => db_user_object)}, &RELATIONSHIP_SAVE)
+  end
+  $SAVER.rules[:complete_follower_set] = false
+  {:follower_relationship_db_ids => follower_relationship_db_ids}
+end
+
+FRIENDS = lambda do |user, auth_object|
+  user.pull_account_info
+  rules = {:user_id => user.twitter_id}
+  friend_relationship_db_ids = []
+  # unless user.desired_friend_count
+  #   $SAVER.rules[:complete_friend_set] = true
+  # end
+  loop do
+    rules[:cursor] = -1 if !rules[:cursor]
+    @pull_data = auth_object.friends(rules)
+    @pull_data.users.each do |friend_mash|
+      db_user_object = $SAVER.save(friend_mash, &TWITTER_ACCOUNT_SAVE)  
+      friend_relationship_db_ids << $SAVER.save({:friend => Pork::TwitterUser.new(:twitter_id => friend_mash.id, :twitter_screen_name => friend_mash.screen_name, :db_object => db_user_object), :follower => user}, &RELATIONSHIP_SAVE).id
+    end
+    if @pull_data.next_cursor == 0
+      break
+    else
+      rules[:cursor] = @pull_data.next_cursor
+    end
+  end
+  {:friend_relationship_db_ids => friend_relationship_db_ids}
+end
+
+FRIEND_IDS = lambda do |user, auth_object|
+  user.pull_account_info
+  # rules[:user_id] = user.twitter_id
+  $SAVER.rules[:complete_friend_set] = true
+  friend_relationship_db_ids = []
+  @pull_data = auth_object.friend_ids({:user_id => user.twitter_id})
+  @pull_data.each do |user_id| 
+    db_user_object = $SAVER.save(Pork::TwitterUser.new(:twitter_id => user_id), &TWITTER_ACCOUNT_SAVE)
+    friend_relationship_db_ids << $SAVER.save({:follower => user, :friend => Pork::TwitterUser.new(:twitter_id => user_id, :db_object => db_user_object)}, &RELATIONSHIP_SAVE)
+  end
+  $SAVER.rules[:complete_friend_set] = false
+  {:friend_relationship_db_ids => friend_relationship_db_ids}
+end
+
+TWEETS = lambda do |user, auth_object|
+  $LOG.info "USER TWEETS PULL"
+  rules = {:count => 200}
+  if user.twitter_id 
+    rules[:user_id] = user.twitter_id
+  else
+    rules[:screen_name] = user.screen_name
+  end
+  @tweet_db_ids = []
+  @pull_data = auth_object.user_timeline(rules)
+  @pull_data.each do |result|
+    @tweet_db_ids << $SAVER.save(result, &USER_TWEET_SAVE).id
+  end
+  # rules[:reactions] ? $REACTION_PROCESSOR.process_reactions(@tweet_db_objects) : nil
+  {:db_ids => @tweet_db_ids}
+end
+
+TRENDS_PULL = lambda do |rules, auth_object|
+  $LOG.info "TRENDS PULL"
+  Twitter::Trends.current().each do |trend|
+    $SAVER.save({:name => trend.name, :query => trend.query}, &TREND_SAVE)
+  end
+end

data/lib/pork/reaction_processor.rb

@@ -0,0 +1,140 @@
+module Pork
+  class ReactionProcessor 
+  attr_accessor :rules, :user
+    def initialize(user, rules = {:pull_secondary_influence_user_info => false})
+      @rules = rules
+      @user = user
+      fill_in_user_info
+    end
+
+    def process_reactions()  
+      if rules[:mention_from] or rules[:reply_from] or rules[:rt_from]
+        if user.twitter_id
+          tweet_ids = $TWITERATOR.twiterate({}, {:user_id => user.twitter_id}, &USER_TWEETS_ITER)[:tweets]
+        else
+          tweet_ids = $TWITERATOR.twiterate({}, {:screen_name => user.twitter_screen_name}, &USER_TWEETS_ITER)[:tweets]        
+        end 
+        if rules[:rt_from]
+          $TWITERATOR.twiterate({}, {:from => user.twitter_screen_name, :search_query => "RT"}, &SEARCH_ITER)[:tweets].each do |tweet_id|
+            tweet_ids << tweet_id
+          end
+        end
+        tweets_from_tweet_ids(tweet_ids).each do |tweet|
+          influentials = parse_tweet_for_influentials(tweet)
+          if rules[:mention_from] 
+            influentials[:mention_screen_names].each do |screen_name|
+              save_from(screen_name, tweet, 'mention')
+            end
+          end
+          if rules[:reply_from] 
+            save_from(influentials[:reply_screen_name], tweet, 'reply')
+          end
+          if rules[:rt_from]
+            influentials[:rt_screen_names].each do |screen_name|
+              save_from(screen_name, tweet, 'retweet')
+            end
+            pull_and_save_rts_from
+          end
+        end
+      end
+      if rules[:mention_to] or rules[:reply_to] or rules[:rt_to]
+        tweet_ids = []
+        $TWITERATOR.twiterate({}, {:search_query => "#{user.twitter_screen_name}"}, &SEARCH_ITER)[:tweets].each do |tweet_id|
+          tweet_ids << tweet_id
+        end
+        tweets_from_tweet_ids(tweet_ids).each do |tweet|
+          source_user_screen_name = tweet.from_user
+          influentials = parse_tweet_for_influentials(tweet)
+          if rules[:mention_to]
+            influentials[:mention_screen_names].each do |screen_name|
+              if screen_name == user.twitter_screen_name
+                save_to(source_user_screen_name, tweet, 'mention')
+              end
+            end
+          end
+          if rules[:reply_to]
+            if influentials[:reply_screen_name] == user.twitter_screen_name
+              save_to(source_user_screen_name, tweet, 'reply')
+            end
+          end
+          if rules[:rt_to]
+            influentials[:rt_screen_names].each do |screen_name|
+              if screen_name == user.twitter_screen_name
+                save_to(source_user_screen_name, tweet, 'retweet')
+              end
+            end
+          end
+        end
+      end
+    end  
+
+
+    def save_from(secondary_screen_name, tweet, type)
+      $SAVER.save({:initiator => user.db_object, :responder => find_or_create_secondary_account(secondary_screen_name), :tweet => tweet, :type => type}, &REACTION_SAVE)
+    end
+
+    def save_to(secondary_screen_name, tweet, type)
+       $SAVER.save({:initiator => find_or_create_secondary_account(secondary_screen_name), :responder => user.db_object, :tweet => tweet, :type => type}, &REACTION_SAVE)
+    end
+
+    def tweets_from_tweet_ids(tweet_ids)
+      tweets = tweet_ids.map do |tweet_id| Tweet.find(tweet_id) end
+    end
+
+ 
+
+    def parse_tweet_for_influentials(tweet) 
+      mention_regex = /[@]\w+/
+      reply_regex = /^[@]\w+/
+      rt_regex = /(^[Rr][Tt] ?[@]\w+(:?)| [Rr][Tt] ?[@]\w+(:?))/  
+      t_copy = tweet.text.dup
+      reply_screen_name = t_copy.scan(reply_regex)
+      reply_screen_name.each do |ru|
+        t_copy.slice!(/^#{ru}/)
+      end
+      rt_screen_names = t_copy.scan(rt_regex)
+      unless rt_screen_names.empty?
+        rt_screen_names.map! do |rt| rt.first end
+      end
+      rt_screen_names.each do |rtu|
+        t_copy.slice!(/^#{rtu}| #{rtu}/)
+      end
+      mention_screen_names = t_copy.scan(mention_regex)
+      sanitize_screen_names(reply_screen_name.first, rt_screen_names, mention_screen_names)
+    end 
+
+    def sanitize_screen_names(reply_screen_name, rt_screen_names, mention_screen_names)
+        sanitizing_regex = / ?([rR][tT])? ?@/
+        reply_screen_name ? reply_screen_name.slice!(sanitizing_regex) : nil
+        rt_screen_names.each do |rtu| 
+          rtu.slice!(sanitizing_regex) 
+        end
+        mention_screen_names.each do |mu| 
+          mu.slice!(sanitizing_regex) 
+        end
+        {:reply_screen_name => reply_screen_name, :rt_screen_names => rt_screen_names, :mention_screen_names => mention_screen_names}
+    end
+  
+    def fill_in_user_info
+      if not user.db_object
+        user.db_object = $PULLER.pull({:user=>user}, &USER_PULL)[:db_object]
+      end
+    end
+  
+    def find_or_create_secondary_account(screen_name)
+      account = TwitterAccount.find_by_screen_name(screen_name)
+      unless account  
+        if rules[:pull_secondary_influence_user_info]
+          account = $PULLER.pull({:user => Pork::TwitterUser.new(:twitter_screen_name => screen_name)})[:db_object]
+        else
+          account = TwitterAccount.create(:screen_name => screen_name)
+        end
+      end
+      account
+    end
+  
+    def pull_and_save_rts_from
+    
+    end
+  end
+end