grailbird_updater 0.1.0 → 0.2.0

data/README.md

@@ -8,6 +8,56 @@ Turns out the contents in the archive are partial/trimmed API responses from
 the Twitter API, so it is actually possible to drop a whole API response in
 there, do some sorting and update the archive.
 
+This currently does not work with a private/protected Twitter account, see 
+[this issue](https://github.com/DeMarko/grailbird_updater/issues/6)
+
+## How do I know if I have a Twitter archive?
+
+Hopefully, you downloaded it from Twitter once the feature was made available
+to you and have their web application which can consume it. The file structure
+looks somewhat like this (as of 19.12.12):
+
+```
+tweets
+├── README.txt
+├── css
+│   └─ ... // provided by Twitter
+├── data
+│   ├── csv
+│   │   ├── 2007_03.csv
+│   │   ├── 2007_04.csv
+│   │   ├── 2007_05.csv
+│   │   ├─ ...
+│   │   ├── 2012_10.csv
+│   │   ├── 2012_11.csv
+│   │   └── 2012_12.csv
+│   └── js
+│       ├── payload_details.js
+│       ├── tweet_index.js
+│       ├── tweets
+│       │   ├── 2007_03.js
+│       │   ├── 2007_04.js
+│       │   ├── 2007_05.js
+│       │   ├─ ... // you get the idea, I've been on Twitter a while
+│       │   ├── 2012_10.js
+│       │   ├── 2012_11.js
+│       │   └── 2012_12.js
+│       └── user_details.js
+├── img
+│   └─ ... // provided by Twitter
+├── index.html
+├── js
+│   └─ ... // provided by Twitter
+└── lib
+    └─ ... // provided by Twitter
+```
+
+This gem only modifies what's in the data directory for a given archive, 
+the rest of the files are provided by Twitter. To check if you can download
+a copy of your Twitter archive, go to your [Account Settings](https://twitter.com/settings/account)
+and scroll all the way to the bottom. If the feature is enabled for you, you should 
+see a section labeled "Your Twitter Archive".
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/bin/grailbird_updater

@@ -1,9 +1,11 @@
 #!/usr/bin/env ruby
 
-require 'open-uri'
+require 'uri'
+require 'net/http'
 require 'json'
-require 'trollop'
+require 'oauth'
 require 'pp'
+require 'trollop'
 require 'colorize' # if verbose
 
 require 'grailbird_updater'

data/grailbird_updater.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.add_dependency "oauth"
   gem.add_dependency "trollop"
   gem.add_dependency "colorize"
 end

data/lib/grailbird_updater/version.rb

@@ -1,3 +1,3 @@
 class GrailbirdUpdater
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/grailbird_updater.rb

@@ -5,8 +5,8 @@ class GrailbirdUpdater
   KEEP_FIELDS = {'user' => ['name', 'screen_name', 'protected', 'id_str', 'profile_image_url_https', 'id', 'verified']}
 
   def initialize(dir, count, verbose, prune)
+    @base_dir = dir
     data_path = dir + "/data"
-
     @js_path = data_path + "/js"
     @csv_path = data_path + "/csv"
 
@@ -37,10 +37,7 @@ class GrailbirdUpdater
 
     vputs "Last tweet in archive is\n\t" + display_tweet(last_tweet)
 
-    # get response from API
-    twitter_url = "http://api.twitter.com/1/statuses/user_timeline.json?count=#{@count}&user_id=#{user_id}&since_id=#{last_tweet_id}&include_rts=true&include_entities=true"
-    vputs "Making request to #{twitter_url}"
-    tweets = JSON.parse(open(twitter_url).read)
+    tweets = JSON.parse(get_twitter_user_timeline_response(screen_name, user_id, last_tweet_id, @count))
 
     vputs "There have been #{tweets.length} tweets since the archive" + (archive_details.has_key?('updated_at') ? " was last updated on #{archive_details['updated_at']}" : " was created")
 
@@ -87,9 +84,105 @@ class GrailbirdUpdater
   def read_twitter_js_file(file_path)
     file_contents = open(file_path).read.force_encoding("UTF-8").split("\n").join(" ")
     json_file_contents = file_contents.gsub(/^((var)?\s*(.+?)\s+=\s+)/m, '')
-    json = JSON.parse(json_file_contents)
+    return JSON.parse(json_file_contents)
+  end
+
+  def get_twitter_user_timeline_response(screen_name, user_id, last_tweet_id, count)
+    twitter_url = "http://api.twitter.com/1/statuses/user_timeline.json"
+    twitter_uri = URI(twitter_url)
+    params = {
+      :count => count,
+      :user_id => user_id,
+      :since_id => last_tweet_id,
+      :include_rts => true,
+      :include_entities => true}
+    twitter_uri.query = URI.encode_www_form(params)
+
+    vputs "Making request to #{twitter_uri}"
+    response = Net::HTTP.get_response(twitter_uri)
+
+    if response.is_a?(Net::HTTPUnauthorized)
+      access_token = do_oauth_dance(screen_name)
+      response = access_token.request(:get, twitter_uri.to_s)
+    end
+
+    return response.body
+  end
+
+  def do_oauth_dance(screen_name)
+    puts "It seems " + "@#{screen_name}".blue + " has a protected account."
+    key_file_path = "#{@base_dir}/#{screen_name}_keys.yaml"
+    if File.exists?(key_file_path)
+        keys = YAML.load_file(key_file_path)
+        consumer_key = keys['consumer_key']
+        consumer_secret = keys['consumer_secret']
+        token = keys['token']
+        token_secret = keys['secret']
+    else
+      puts <<-EOS
+          To be able to retrieve your protected tweets, you will need a consumer key/secret
+
+          Please follow these steps to authorize grailbird_updater to download tweets:
+              1. Go to https://dev.twitter.com/apps/new
+              2. Give it a name (I recommend #{screen_name}_grailbird), description and URL
+              3. Create application
+              4. Go to your application page, you should see a "Consumer key" and a "Consumer secret"
+
+          Note: you will only need to create this application once!
+
+          So you don't have to enter these again, we'll save a copy of your keys in a file called #{screen_name}_keys.yaml
+
+          IMPORTANT: Do NOT store the folder of your tweets on a public server. If someone gets access to #{screen_name}_keys.yaml they can access your entire account!
+          Did I mention that was IMPORTANT? BECAUSE IT IS.
+      EOS
+
+      puts "Enter your 'Consumer key'"
+      consumer_key = STDIN.gets.chomp
+      puts "Enter your 'Consumer secret'"
+      consumer_secret = STDIN.gets.chomp
+      consumer = OAuth::Consumer.new(
+        consumer_key,
+        consumer_secret,
+        { :site => 'http://api.twitter.com/',
+          :request_token_path => '/oauth/request_token',
+          :access_token_path => '/oauth/access_token',
+          :authorize_path => '/oauth/authorize' }
+      )
+      request_token = consumer.get_request_token
+      puts "Go to this URL: #{request_token.authorize_url()}"
+      puts "Authorize the application and you will receive a PIN"
+      puts "Enter the PIN here:"
+      pin = STDIN.gets.chomp
+      access_token = request_token.get_access_token(:oauth_verifier => pin)
+
+      token = access_token.token
+      token_secret = access_token.secret
+      tokens = {
+        'consumer_key' => consumer_key,
+        'consumer_secret' => consumer_secret,
+        'token' => token,
+        'secret' => token_secret}
+      File.open(key_file_path, 'w+') {|f| f.write(tokens.to_yaml) }
+    end
+
+    # Exchange our oauth_token and oauth_token secret for the AccessToken instance.
+    access_token = prepare_access_token(consumer_key, consumer_secret, token, token_secret)
   end
 
+  def prepare_access_token(consumer_key, consumer_secret, oauth_token, oauth_token_secret)
+    consumer = OAuth::Consumer.new(consumer_key, consumer_secret,
+      { :site => "http://api.twitter.com",
+        :scheme => :header
+      })
+    # now create the access token object from passed values
+    token_hash = { :oauth_token => oauth_token,
+                  :oauth_token_secret => oauth_token_secret
+                }
+    access_token = OAuth::AccessToken.from_hash(consumer, token_hash )
+    return access_token
+  end
+
+
   def prune_tweet(tweet)
     KEEP_FIELDS.each do |parent_field, field_names|
       tweet[parent_field].delete_if { |key, value| !field_names.include?(key) }
@@ -124,7 +217,7 @@ class GrailbirdUpdater
         "tweet_count" => count,
         "month" => month
     }
-    new_index = tweet_index.unshift(new_month).sort_by {|m| [-m['year'], -m['month']]}
+    return tweet_index.unshift(new_month).sort_by {|m| [-m['year'], -m['month']]}
   end
 
   def write_twitter_js_to_path_with_heading(contents, path, heading)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: grailbird_updater
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-19 00:00:00.000000000 Z
+date: 2012-12-23 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: oauth
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: trollop
   requirement: !ruby/object:Gem::Requirement