grailbird_updater 0.1.0

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+
+# gems need this ignored
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in grailbird.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Dannel Jurado
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# GrailbirdUpdater
+
+For the most of the people who know me online, I've been dying to get a copy of
+my Twitter archive from Twitter for forever. I was finally given one and
+decided to write a quick script to keep my own archive up-to-date.
+
+Turns out the contents in the archive are partial/trimmed API responses from
+the Twitter API, so it is actually possible to drop a whole API response in
+there, do some sorting and update the archive.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'grailbird_updater'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install grailbird_updater
+
+## Usage
+
+```
+grailbird_updater /path/to/twitter/archive
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "lib"
+  t.test_files = FileList['test/*_test.rb']
+  t.ruby_opts = ['-r./test/test_helper.rb']
+  t.verbose = true
+end
+
+task :default => :test

data/bin/grailbird_updater

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+require 'open-uri'
+require 'json'
+require 'trollop'
+require 'pp'
+require 'colorize' # if verbose
+
+require 'grailbird_updater'
+
+opts = Trollop::options do
+    version "updater #{GrailbirdUpdater::VERSION}"
+    banner <<-EOS
+Update your Twitter archive (best if used with a cron)
+
+Usage: updater [options] [path to archive]
+
+EOS
+    opt :verbose, "Verbose mode"
+    opt :prune, "Prune all but necessary user data from tweets", :default => true
+    opt :limit, "How many tweets to look back at (max: 3200)", :default => 1000
+    opt :directory, "Twitter archive directory", :type => :string
+end
+
+dir = nil
+dir ||= opts[:directory]
+dir ||= ARGV.first
+dir ||= "."
+raise ArgumentError, "Must specify a directory" unless File.directory?(dir)
+raise ArgumentError, "Cannot look back further than 3200 tweets" if opts[:limit] > 3200
+
+GrailbirdUpdater.new(dir, opts[:limit], opts[:verbose], opts[:prune]).update_tweets

data/grailbird_updater.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'grailbird_updater/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "grailbird_updater"
+  gem.version       = GrailbirdUpdater::VERSION
+  gem.authors       = ["Dannel Jurado"]
+  gem.email         = ["dannelj@gmail.com"]
+  gem.description   = %q{Twitter now allows you to download your tweets. This tool lets you keep that archive up to date.}
+  gem.summary       = %q{A way to keep an updated archive of Twitter tweets.}
+  gem.homepage      = "https://github.com/DeMarko/grailbird_updater"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "trollop"
+  gem.add_dependency "colorize"
+end

data/lib/grailbird_updater/version.rb

@@ -0,0 +1,3 @@
+class GrailbirdUpdater
+  VERSION = "0.1.0"
+end

data/lib/grailbird_updater.rb

@@ -0,0 +1,142 @@
+require "grailbird_updater/version"
+
+class GrailbirdUpdater
+
+  KEEP_FIELDS = {'user' => ['name', 'screen_name', 'protected', 'id_str', 'profile_image_url_https', 'id', 'verified']}
+
+  def initialize(dir, count, verbose, prune)
+    data_path = dir + "/data"
+
+    @js_path = data_path + "/js"
+    @csv_path = data_path + "/csv"
+
+    @count = count
+    @verbose = verbose
+    @prune = prune
+  end
+
+  def update_tweets
+    # find user_id in data/js/user_details.js
+    user_details = read_required_twitter_js_file("#{@js_path}/user_details.js")
+    user_id = user_details["id"]
+    screen_name = user_details["screen_name"]
+    vputs "Twitter Archive for " + "@#{screen_name}".light_blue + " (##{user_id}) found"
+
+    # find archive details
+    archive_details = read_required_twitter_js_file("#{@js_path}/payload_details.js")
+    vputs "Found archive payload containing #{archive_details['tweets']} tweets, created at #{archive_details['created_at']}"
+
+    # find latest month file (should be last when sorted alphanumerically)
+    twitter_js_files = Dir.glob("#{@js_path}/tweets/*.js")
+    latest_month = read_required_twitter_js_file(twitter_js_files.sort.last)
+
+    # find last_tweet_id in latest_month (should be first, because Twitter)
+    last_tweet = latest_month.first
+    last_tweet_id = last_tweet["id_str"]
+    last_tweet_date = Date.parse(last_tweet["created_at"])
+
+    vputs "Last tweet in archive is\n\t" + display_tweet(last_tweet)
+
+    # get response from API
+    twitter_url = "http://api.twitter.com/1/statuses/user_timeline.json?count=#{@count}&user_id=#{user_id}&since_id=#{last_tweet_id}&include_rts=true&include_entities=true"
+    vputs "Making request to #{twitter_url}"
+    tweets = JSON.parse(open(twitter_url).read)
+
+    vputs "There have been #{tweets.length} tweets since the archive" + (archive_details.has_key?('updated_at') ? " was last updated on #{archive_details['updated_at']}" : " was created")
+
+    # collect tweets by year_month
+    collected_months = Hash.new
+    tweets.each do |tweet|
+      tweet = prune_tweet(tweet) if @prune
+      vputs "\t" + display_tweet(tweet)
+      tweet_date = Date.parse(tweet["created_at"])
+      hash_index = tweet_date.strftime('%Y_%m')
+      collected_months[hash_index] = Array(collected_months[hash_index])
+      collected_months[hash_index] << tweet
+    end
+
+    # add tweets to json data file
+    tweet_index = read_required_twitter_js_file("#{@js_path}/tweet_index.js")
+    collected_months.each do |year_month, month_tweets|
+      month_path = "#{@js_path}/tweets/#{year_month}.js"
+
+      existing_month_tweets = (File.exists?(month_path)) ? read_twitter_js_file(month_path) : []
+      all_month_tweets = month_tweets | existing_month_tweets
+      # sort new collection of tweets for this month by reverse date
+      all_month_tweets.sort_by {|t| -Date.parse(t['created_at']).strftime("%s").to_i }
+
+      # overwrite existing file (or create new if doesn't exist)
+      write_twitter_js_to_path_with_heading(all_month_tweets, "#{@js_path}/tweets/#{year_month}.js", "Grailbird.data.tweets_#{year_month}")
+      tweet_index = update_tweet_index(tweet_index, year_month, month_tweets.length)
+    end
+
+    # write new tweet_index.js once
+    write_twitter_js_to_path_with_heading(tweet_index, "#{@js_path}/tweet_index.js", "var tweet_index")
+
+    # add count to payload_details.js
+    archive_details['tweets'] += tweets.length
+    archive_details['updated_at'] = Time.now.getgm.strftime("%a %b %d %T %z %Y")
+    write_twitter_js_to_path_with_heading(archive_details, "#{@js_path}/payload_details.js", "var payload_details")
+  end
+
+  def read_required_twitter_js_file(file_path)
+    raise "#{file_path} must exist" unless  File.exists?(file_path)
+    read_twitter_js_file(file_path)
+  end
+
+  def read_twitter_js_file(file_path)
+    file_contents = open(file_path).read.force_encoding("UTF-8").split("\n").join(" ")
+    json_file_contents = file_contents.gsub(/^((var)?\s*(.+?)\s+=\s+)/m, '')
+    json = JSON.parse(json_file_contents)
+  end
+
+  def prune_tweet(tweet)
+    KEEP_FIELDS.each do |parent_field, field_names|
+      tweet[parent_field].delete_if { |key, value| !field_names.include?(key) }
+    end
+    return tweet
+  end
+
+  def display_tweet(tweet)
+    tweet_text = tweet['text']
+    if tweet['entities'] && tweet['entities']['urls']
+      tweet['entities']['urls'].each { |url_entity|
+        tweet_text = tweet['text'].gsub("#{url_entity['url']}", "#{url_entity['expanded_url']}")
+      }
+    end
+    tweet = "@#{tweet['user']['screen_name']}".blue + ": \"#{tweet_text}\"\n"
+  end
+
+  def update_tweet_index(tweet_index, year_month, count)
+    year, month = year_month.split('_')
+    year = year.to_i
+    month = month.to_i
+    tweet_index.each do |index_month|
+      if index_month['year'] == year && index_month['month'] == month
+        index_month['tweet_count'] += count
+        return tweet_index
+      end
+    end
+
+    new_month = {"file_name" => "data/js/tweets/#{year_month}.js",
+      "year" => year,
+        "var_name" => "tweets_#{year_month}",
+        "tweet_count" => count,
+        "month" => month
+    }
+    new_index = tweet_index.unshift(new_month).sort_by {|m| [-m['year'], -m['month']]}
+  end
+
+  def write_twitter_js_to_path_with_heading(contents, path, heading)
+    json_pretty_contents = JSON.pretty_generate(contents)
+    File.open(path, 'w') {|f| f.write("#{heading} = #{json_pretty_contents}")}
+  end
+
+  private
+
+  # only puts if we're verbose
+  def vputs(str)
+    puts str if @verbose
+  end
+end
+

data/test/grailbird_updater_test.rb

@@ -0,0 +1,12 @@
+require 'minitest/autorun'
+
+require 'grailbird_updater'
+
+class GrailbirdUpdaterTest < Minitest::Unit::TestCase
+
+  # this test is stupid, just there to demonstrate infrastructure
+  def test_creation
+    assert_kind_of GrailbirdUpdater, GrailbirdUpdater.new(".", 10, true)
+  end
+end
+

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: grailbird_updater
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Dannel Jurado
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Twitter now allows you to download your tweets. This tool lets you keep
+  that archive up to date.
+email:
+- dannelj@gmail.com
+executables:
+- grailbird_updater
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/grailbird_updater
+- grailbird_updater.gemspec
+- lib/grailbird_updater.rb
+- lib/grailbird_updater/version.rb
+- test/grailbird_updater_test.rb
+- test/test_helper.rb
+homepage: https://github.com/DeMarko/grailbird_updater
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: A way to keep an updated archive of Twitter tweets.
+test_files:
+- test/grailbird_updater_test.rb
+- test/test_helper.rb