grailbird_updater 0.4.0 → 0.5.0

data/bin/grailbird_updater

@@ -2,11 +2,12 @@
 
 require 'uri'
 require 'net/http'
+require 'yaml'
 require 'json'
+require 'csv'
 require 'oauth'
-require 'pp'
 require 'trollop'
-require 'colorize' # if verbose
+require 'colorize'
 
 require 'grailbird_updater'
 
@@ -17,10 +18,14 @@ Update your Twitter archive (best if used with a cron)
 
 Usage: updater [options] [path to archive]
 
+Where [options] can be:
 EOS
+
     opt :verbose, "Verbose mode"
     opt :prune, "Prune all but necessary user data from tweets", :default => true
+    opt :csv_write, "Also write to CSV files in archive"
     opt :directory, "Twitter archive directory", :type => :string
+    opt :key_path, "Alternate path to directory containing a YAML file with the app/user keys for the archive's account. Defaults to the archive directory", :type => :string
 end
 
 dir = nil
@@ -29,4 +34,11 @@ dir ||= ARGV.first
 dir ||= "."
 raise ArgumentError, "Must specify a directory" unless File.directory?(dir)
 
-GrailbirdUpdater.new(dir, opts[:verbose], opts[:prune]).update_tweets
+key_path = opts[:key_path]
+key_path ||= dir
+
+raise ArgumentError, "#{key_path} is not a directory" unless File.directory?(key_path)
+raise IOError, "#{key_path} is not readable" unless File.readable?(key_path)
+raise IOError, "#{key_path} is not writable" unless File.writable?(key_path)
+
+GrailbirdUpdater.new(dir, opts[:verbose], opts[:prune], key_path, opts[:csv_write]).update_tweets

data/grailbird_updater.gemspec

@@ -20,4 +20,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency "oauth"
   gem.add_dependency "trollop"
   gem.add_dependency "colorize"
+  gem.add_dependency "minitest"
 end

data/lib/grailbird_updater.rb

@@ -4,6 +4,7 @@ class GrailbirdUpdater
 
   KEEP_FIELDS = {'user' => ['name', 'screen_name', 'protected', 'id_str', 'profile_image_url_https', 'id', 'verified']}
   MAX_REQUEST_SIZE = 200
+  PLATFORM_IS_OSX = (Object::RUBY_PLATFORM =~ /darwin/i) ? true : false
 
   class JsFile
     # Read UTF-8 file and return hash of contents (files being read contain JS arrays)
@@ -20,7 +21,7 @@ class GrailbirdUpdater
     # @param file_path [String] path to file being read
     # @raise [IOError] if the required file isn't found
     def self.read_required(file_path)
-      raise IOError "#{file_path} must exist" unless File.exists?(file_path)
+      raise IOError, "#{file_path} must exist" unless File.exists?(file_path)
       read(file_path)
     end
 
@@ -35,12 +36,50 @@ class GrailbirdUpdater
     end
   end
 
-  def initialize(dir, verbose, prune)
+  class CsvFile
+    # Write Twitter's archive CSV files
+    # @param tweets [Array] all of the tweets you want to write to the file
+    # @param file_path [String] path to file being written
+    def self.write_tweets_csv (tweets, csv_path)
+      CSV.open(csv_path, "w") do |csv|
+        csv << ["tweet_id", "in_reply_to_status_id", "in_reply_to_user_id", "retweeted_status_id", "retweeted_status_user_id", "timestamp", "source", "text", "expanded_urls"]
+        tweets.each do |tweet|
+          csv << parse_tweet_into_csv_array(tweet)
+        end
+      end
+    end
+
+    # Auxiliary function that turns a Tweet hash array (a single tweet from the API,
+    # encoded as a Hash) into the Array to write out to Twitter's CSV
+    #
+    # @param tweet [Hash] single tweet, encoded as a Hash
+    # @return [Array] the tweet as an array
+    def self.parse_tweet_into_csv_array (tweet)
+      csv_tweet_array = [tweet["id"],
+                        tweet["in_reply_to_status_id"],
+                        tweet["in_reply_to_user_id"],
+                        tweet.has_key?("retweeted_status") ? tweet["retweeted_status"]["id"] : '',
+                        tweet.has_key?("retweeted_status") ? tweet["retweeted_status"]["user"]["id"] : '',
+                        tweet["created_at"],
+                        tweet["source"],
+                        tweet["text"]];
+      if tweet.has_key?("entities") && tweet["entities"].has_key?("urls")
+          tweet["entities"]["urls"].each do |url|
+              csv_tweet_array << url["expanded_url"]
+          end
+      end
+      return csv_tweet_array
+    end
+  end
+
+  def initialize(dir, verbose, prune, key_dir, write_csv)
     @base_dir = dir
     data_path = dir + "/data"
     @js_path = data_path + "/js"
     @csv_path = data_path + "/csv"
+    @key_path = key_dir
 
+    @write_csv = write_csv
     @verbose = verbose
     @prune = prune
     @access_token = nil
@@ -94,6 +133,7 @@ class GrailbirdUpdater
 
       # overwrite existing file (or create new if doesn't exist)
       GrailbirdUpdater::JsFile.write_with_heading(all_month_tweets, "#{@js_path}/tweets/#{year_month}.js", "Grailbird.data.tweets_#{year_month}")
+      GrailbirdUpdater::CsvFile.write_tweets_csv(all_month_tweets, "#{@csv_path}/#{year_month}.csv") if @write_csv
       tweet_index = update_tweet_index(tweet_index, year_month, month_tweets.length)
     end
 
@@ -107,7 +147,7 @@ class GrailbirdUpdater
   end
 
   def get_twitter_user_timeline_response(screen_name, user_id, last_tweet_id)
-    twitter_url = "http://api.twitter.com/1/statuses/user_timeline.json"
+    twitter_url = "http://api.twitter.com/1.1/statuses/user_timeline.json"
     twitter_uri = URI(twitter_url)
 
     params = {
@@ -143,17 +183,13 @@ class GrailbirdUpdater
     if !@access_token.nil?
       response = @access_token.request(:get, twitter_uri.to_s)
     else
-      response = Net::HTTP.get_response(twitter_uri)
-
-      if response.is_a?(Net::HTTPUnauthorized)
-        @access_token = do_oauth_dance(screen_name)
-        response = @access_token.request(:get, twitter_uri.to_s)
-      end
+      @access_token = do_oauth_dance(screen_name)
+      response = @access_token.request(:get, twitter_uri.to_s)
     end
 
     if response.is_a?(Net::HTTPUnauthorized)
       puts "\nSomething went wrong trying to authorize grailbird_updater with the account: " + "@#{screen_name}".blue
-      puts "Please delete #{@base_dir}/#{screen_name}_keys.yaml and follow the authorize steps again."
+      puts "Please delete #{@key_path}/#{screen_name}_keys.yaml and follow the authorize steps again."
       exit
     end
 
@@ -161,8 +197,8 @@ class GrailbirdUpdater
   end
 
   def do_oauth_dance(screen_name)
-    puts "\nIt seems " + "@#{screen_name}".blue + " has a protected account."
-    key_file_path = "#{@base_dir}/#{screen_name}_keys.yaml"
+    key_file_path = "#{@key_path}/#{screen_name}_keys.yaml"
+
     if File.exists?(key_file_path)
         keys = YAML.load_file(key_file_path)
         consumer_key = keys['consumer_key']
@@ -171,25 +207,31 @@ class GrailbirdUpdater
         token_secret = keys['secret']
     else
       puts <<-EOS
-      To be able to retrieve your protected tweets, you will need a consumer key/secret
+\nTo be able to retrieve your protected tweets, you will need a consumer key/secret
+
+Please follow these steps to authorize grailbird_updater to download tweets:
+    1. Go to https://dev.twitter.com/apps/new
+    2. Give it a name (I recommend #{screen_name}_grailbird), description and URL
+    3. Create application
+    4. Go to your application page, you should see a "Consumer key" and a "Consumer secret"
+    5. Enter these here when prompted, go to the URL provided then enter the PIN you receive
 
-      Please follow these steps to authorize grailbird_updater to download tweets:
-          1. Go to https://dev.twitter.com/apps/new
-          2. Give it a name (I recommend #{screen_name}_grailbird), description and URL
-          3. Create application
-          4. Go to your application page, you should see a "Consumer key" and a "Consumer secret"
+#{"Note".underline}: you will only need to create this application once!
 
-      Note: you will only need to create this application once!
+So you don't have to enter these again, we'll save a copy of your keys to:
+    #{key_file_path}
 
-      So you don't have to enter these again, we'll save a copy of your keys in a file called #{screen_name}_keys.yaml
+You can always change the directory these are saved to by using the -k or --key-path option
 
-      #{"IMPORTANT".red.blink} Do NOT store the folder of your tweets on a public server.
-                If someone gets access to #{screen_name}_keys.yaml they can access your entire account!
-      EOS
+#{"WARNING".red.underline} Do NOT store the folder of your tweets on a public server.
+    If someone gets access to #{screen_name}_keys.yaml they can access your entire account!
+    If you want to share your archived tweets, either control the read access to the key file
+    OR use the --key-path option to store them somewhere else.
+EOS
 
-      puts "\nEnter your 'Consumer key'"
+      print_flush "\nEnter your 'Consumer key': "
       consumer_key = STDIN.gets.chomp
-      puts "Enter your 'Consumer secret'"
+      print_flush "Enter your 'Consumer secret': "
       consumer_secret = STDIN.gets.chomp
       consumer = OAuth::Consumer.new(
         consumer_key,
@@ -200,9 +242,16 @@ class GrailbirdUpdater
           :authorize_path => '/oauth/authorize' }
       )
       request_token = consumer.get_request_token
-      puts "\nGo to this URL: #{request_token.authorize_url()}"
+      authorize_url = request_token.authorize_url()
+      puts "\nGo to this URL: #{authorize_url}"
       puts "Authorize the application and you will receive a PIN"
-      puts "Enter the PIN here:"
+      # open default browser if on OS X
+      if PLATFORM_IS_OSX
+        sleep(2)
+        `open "#{authorize_url}"`
+      end
+
+      print_flush "Enter the PIN here: "
       pin = STDIN.gets.chomp
       access_token = request_token.get_access_token(:oauth_verifier => pin)
 
@@ -226,7 +275,7 @@ class GrailbirdUpdater
         :scheme => :header
       })
     # now create the access token object from passed values
-    token_hash = { :oauth_token => oauth_token,
+    token_hash = {:oauth_token => oauth_token,
                   :oauth_token_secret => oauth_token_secret
                 }
     access_token = OAuth::AccessToken.from_hash(consumer, token_hash )
@@ -241,13 +290,14 @@ class GrailbirdUpdater
   end
 
   def display_tweet(tweet)
+    tweet = tweet["retweeted_status"] if tweet.has_key?("retweeted_status")
     tweet_text = tweet['text']
     if tweet['entities'] && tweet['entities']['urls']
       tweet['entities']['urls'].each { |url_entity|
         tweet_text = tweet['text'].gsub("#{url_entity['url']}", "#{url_entity['expanded_url']}")
       }
     end
-    tweet = "@#{tweet['user']['screen_name']}".blue + ": \"#{tweet_text}\"\n"
+    tweet = "@#{tweet['user']['screen_name']}".blue + ": #{tweet_text}\n"
   end
 
   def update_tweet_index(tweet_index, year_month, count)
@@ -262,7 +312,7 @@ class GrailbirdUpdater
     end
 
     new_month = {"file_name" => "data/js/tweets/#{year_month}.js",
-      "year" => year,
+        "year" => year,
         "var_name" => "tweets_#{year_month}",
         "tweet_count" => count,
         "month" => month
@@ -276,5 +326,10 @@ class GrailbirdUpdater
   def vputs(str)
     puts str if @verbose
   end
+
+  def print_flush(str)
+    print str
+    $stdout.flush
+  end
 end
 

data/lib/grailbird_updater/version.rb

@@ -1,3 +1,3 @@
 class GrailbirdUpdater
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: grailbird_updater
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-10 00:00:00.000000000 Z
+date: 2013-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oauth
@@ -59,6 +59,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Twitter now allows you to download your tweets. This tool lets you keep
   that archive up to date.
 email:
@@ -106,4 +122,3 @@ summary: A way to keep an updated archive of Twitter tweets.
 test_files:
 - test/grailbird_updater_test.rb
 - test/test_helper.rb
-has_rdoc: