twitter_backup 0.1.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in twitter_backup.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 il.zoff
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# TwitterBackup
+
+This gem will download your tweets from Twitter and save them to sqlite3 database and plaintext (yaml) archive file.
+
+## Installation
+
+    $ gem install twitter_backup
+
+## Usage
+
+First of all, you need to create an app at https://dev.twitter.com/apps
+Your app will be assigned a 'consumer key'/'consumer secret' pair and you as a user will be assigned an 'access token'/'acces token secret' OAuth pair for that application.
+Without these credentials you won't be able to use TwitterBackup.
+
+Now, all you have to do is to run
+
+    $ twitter_backup
+
+If you want to know, what's happening during your backup process, use verbose mode with `-v`.
+
+    $ twitter_backup -h
+        -v, --verbose      Enable verbose mode
+        -f, --force        Try to download tweets, even if it seems useless
+        -s, --seed         Try to download tweets older than the oldest one you have
+        -c, --config       Config file. Default: ~/.config/twitter_backup/config.yml
+        -h, --help         Display this help message.
+
+## TODO
+
+- Automate adding this script to cron?

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/twitter_backup

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+# require File.expand_path(File.join(File.dirname(__FILE__), %w[.. lib twitter_backup]))
+require "twitter_backup"
+require "twitter_backup/ui"
+
+include TwitterBackup
+
+if TBConfig.passed_opts.help?
+  TBConfig.passed_opts.help 
+  exit 0
+end
+
+begin
+  
+  say ".... checking existance of credentials" if TBConfig.passed_opts.verbose?
+  UI.ask_credentials if TBConfig.credentials_missing?
+
+  UI.define_path_to_backup unless TBConfig.path_to_backup_defined?
+
+  TBConfig.configure
+
+  UI.greet_user
+
+  Tweet.update_tweets
+  Tweet.update_tweets(:from => :earliest) if TBConfig.passed_opts.seed?
+
+  Tweet.dump_to_backup_file
+
+rescue Error::MissingCredentials => error
+  UI.missing_credentials_exit
+
+rescue Twitter::Error::Unauthorized => error
+  TBConfig.empty_credentials
+  UI.wrong_credentials_exit
+
+rescue Twitter::Error::TooManyRequests => error
+  UI.too_many_requests_exit
+
+rescue Error::InvalidBackupFile => error
+  UI.failed_backup_path_exit( error.message )
+
+rescue Error::InvalidPath => error
+  say %[<%= color("#{error.message}", :red) %>]
+  exit 1
+ensure
+  UI.exit_screen
+end

data/lib/twitter_backup.rb

@@ -0,0 +1,47 @@
+require "slop"
+require "yaml"
+require "fileutils"
+require "highline/import"
+require 'active_support/core_ext/string'
+require "active_record"
+require "twitter"
+require_relative "twitter_backup/version"
+require_relative "twitter_backup/error"
+require_relative "twitter_backup/config"
+require_relative "twitter_backup/tweet"
+
+module TwitterBackup
+  def self.prepare_file file
+    file = File.expand_path file
+    dir = File.dirname file
+    FileUtils.mkdir_p dir unless File.exist? dir
+    FileUtils.touch file unless File.exist? file
+    raise( Error::InvalidPath, "Can't create file: #{file}" ) unless File.exist?( file )
+  end
+end
+
+unless Kernel.respond_to?(:require_relative)
+  module Kernel
+    def require_relative(path)
+      require File.join(File.dirname(caller[0]), path.to_str)
+    end
+  end
+end
+class File
+  def tail(n)
+    buffer = 1024
+    idx = (size - buffer).abs
+    chunks = []
+    lines = 0
+    begin
+      seek(idx)
+      chunk = read(buffer)
+      lines += chunk ? chunk.count("\n") : 0
+      chunks.unshift chunk
+      idx -= buffer
+    end while lines < ( n + 1 ) && pos != 0
+    tail_of_file = chunks.join('')
+    ary = tail_of_file.split(/\n/)
+    lines_to_return = ary[ ary.size - n, ary.size - 1 ] || []
+  end
+end

data/lib/twitter_backup/config.rb

@@ -0,0 +1,138 @@
+
+CONFIG_DIR = File.expand_path(File.join("~/", ".config", "twitter_backup"))
+CONFIG_FILE = File.join(CONFIG_DIR, "config.yml")
+
+CONFIG_DEFAULTS = {
+  :credentials => {
+    :consumer_key => "",
+    :consumer_secret => "",
+    :oauth_token => "",
+    :oauth_token_secret => "",
+    },
+  :db => {
+    :adapter => "sqlite3",
+    :database => File.join(CONFIG_DIR, "tweets.sqlite3"),
+    :pool => 5,
+    :timeout => 5000
+    },
+  :backup_file => ""
+}
+
+module TwitterBackup
+  class TBConfig
+    class << self
+      def load
+        TwitterBackup.prepare_file config_file
+        yaml = YAML::load_file( config_file )
+        @@options = if yaml.is_a? Hash
+          CONFIG_DEFAULTS.merge(yaml)
+        else
+          CONFIG_DEFAULTS
+        end
+      end
+
+      def save options
+        @@options = options
+        TwitterBackup.prepare_file config_file
+        File.open( config_file, "w" ){|f| YAML::dump( options, f )}
+      end
+
+      def options
+        @@options ||= load
+      end
+
+      def credentials_missing?
+        options[:credentials].values.map(&:strip).include? ""
+      end
+
+      def path_to_backup_defined?
+        options[:backup_file].present?
+      end
+
+      def save_backup_file file
+        options[:backup_file] = file
+        save options
+      end
+
+      def save_credentials new_credentials
+        options[:credentials] = options[:credentials].merge(new_credentials)
+        save options
+      end
+
+      def empty_credentials
+        new_credentials = { :consumer_key => "",
+                            :consumer_secret => "",
+                            :oauth_token => "",
+                            :oauth_token_secret => ""}
+        save_credentials new_credentials
+      end
+
+      def configure_database
+        ActiveRecord::Base.establish_connection(options[:db])
+        unless ActiveRecord::Base.connection.tables.include?("tweets")
+          ActiveRecord::Migration.create_table :tweets do |t|
+            t.text :status
+            t.integer :status_id
+            t.timestamp :created_at
+            t.text :raw
+          end
+        end
+      end
+
+      def configure_twitter_gem
+        raise MissingCredentials if credentials_missing?
+        Twitter.configure do |config|
+          config.consumer_key = options[:credentials][:consumer_key]
+          config.consumer_secret = options[:credentials][:consumer_secret]
+          config.oauth_token = options[:credentials][:oauth_token]
+          config.oauth_token_secret = options[:credentials][:oauth_token_secret]
+        end
+        @@twitter_gem_configured = true
+      end
+
+      def check_backup_file
+        begin 
+          backup_file = File.expand_path(options[:backup_file])
+          TwitterBackup.prepare_file( backup_file ) unless File.exist?( backup_file )
+        rescue Exception => e
+          raise Error::InvalidBackupFile, e.message
+        end
+      end
+
+      def configure
+        say ".... configuring database" if passed_opts.verbose?
+        configure_database
+
+        say ".... configuring twitter gem" if passed_opts.verbose?
+        configure_twitter_gem
+
+        say ".... checking backup file: #{options[:backup_file]}" if passed_opts.verbose?
+        check_backup_file
+      end
+
+      def verify_credentials
+        say ".... checking validity of credentials" if passed_opts.verbose?
+        configure_twitter_gem unless @@twitter_gem_configured
+        @@user = Twitter.verify_credentials(:skip_status => true, :include_entities => false)
+      end
+
+      def user
+        @@user ||= verify_credentials
+      end
+
+      def config_file
+        passed_opts[:config] || CONFIG_FILE
+      end
+
+      def passed_opts 
+        @@opts ||= Slop.parse(:help => true) do
+          on :v, :verbose, 'Enable verbose mode'
+          on :f, :force, "Try to download tweets, even it seems useless"
+          on :s, :seed, "Try to download tweets older then the oldest one you have"
+          on :c, :config, "Config file. Default: #{CONFIG_FILE}", :argument => true
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_backup/error.rb

@@ -0,0 +1,7 @@
+module TwitterBackup
+  module Error
+    class InvalidPath < StandardError; end
+    class InvalidBackupFile < StandardError; end
+    class MissingCredentials < StandardError; end
+  end
+end

data/lib/twitter_backup/tweet.rb

@@ -0,0 +1,116 @@
+module TwitterBackup
+  class Tweet < ActiveRecord::Base
+
+    serialize :raw
+
+    scope :slim, select([:id, :created_at, :status, :status_id])
+    
+    class << self
+
+      def update_tweets(args={})
+        args = {
+          :from    => nil,
+          :to    => latest.try(:status_id)
+        }.merge(args)
+        if args[:from] == :earliest
+          args[:from] = earliest.try(:status_id)
+          args[:to] = nil if args[:to] == latest.try(:status_id)
+        end
+        if !synced? || TBConfig.passed_opts.force?
+          say ".... Updating tweets from #{args[:from] || 'the youngest one'} to #{args[:to] || 'the oldest one'}" if TBConfig.passed_opts.verbose?
+          download(:from => args[:from], :to => args[:to]).each do |tweet|
+            if absent_tweet = find_by_status_id(tweet.id)
+              next
+            else
+              say %[<%= color(".", YELLOW) %> ] if TBConfig.passed_opts.verbose?
+              tweet_text = if tweet.retweet?
+                "RT @#{tweet.retweeted_status.user.screen_name}: #{tweet.retweeted_status.text}"
+              else
+                tweet.text
+              end
+              create(
+                :status_id => tweet.id,
+                :status => tweet_text,
+                :created_at => tweet.created_at,
+                :raw => tweet
+              )
+            end
+          end
+          say "!"
+          say ".... Updating succeeded" if TBConfig.passed_opts.verbose?
+        else
+          say ".... Seems like there is nothing we can update" if TBConfig.passed_opts.verbose?
+        end
+      end
+
+      def latest
+        self.slim.order("created_at DESC").first
+      end
+      def earliest
+        self.slim.order("created_at DESC").last
+      end
+
+      def download(args={})
+        args = {
+          :from    => nil,
+          :to    => nil
+        }.merge(args)
+        args[:from], args[:to] = [args[:from], args[:to]].sort.reverse unless [args[:from], args[:to]].include? nil
+
+        needed_tweets = []
+
+        if args[:from].blank?
+          say ".... user_timeline API request" if TBConfig.passed_opts.verbose?
+          received_tweets = Twitter.user_timeline(:count => 200)
+        else
+          say ".... user_timeline API request" if TBConfig.passed_opts.verbose?
+          received_tweets = Twitter.user_timeline(:count => 200, :max_id => args[:from])
+        end
+
+        if args[:to].blank?
+          id_of_the_earliest_received_tweet = nil
+          until id_of_the_earliest_received_tweet == received_tweets.last.id
+            id_of_the_earliest_received_tweet = received_tweets.last.id
+            say ".... user_timeline API request" if TBConfig.passed_opts.verbose?
+            received_tweets.concat(
+                Twitter.user_timeline(:count => 200, :max_id => received_tweets.last.id)
+            )
+          end
+          received_tweets
+        else args[:to]
+          id_of_the_earliest_needed_tweet = args[:to]
+          until received_tweets.map(&:id).include? id_of_the_earliest_needed_tweet
+            say ".... user_timeline API request" if TBConfig.passed_opts.verbose?
+            received_tweets.concat(
+                Twitter.user_timeline(:count => 200, :max_id => received_tweets.last.id)
+            )
+          end
+          index_of_earliest_needed_tweet = received_tweets.rindex{|tweet| tweet.id == id_of_the_earliest_needed_tweet}
+          received_tweets[0..index_of_earliest_needed_tweet]
+        end
+
+      end
+
+      def synced?
+        self.count == TBConfig.user.statuses_count
+      end
+
+      def dump_to_backup_file
+        if TBConfig.passed_opts.verbose?
+          say ".... Saving tweets to: #{TBConfig.options[:backup_file]}"
+        end
+        tweets = slim.order("created_at DESC").map{ |tweet| {
+                                          :id => tweet.status_id,
+                                          :text => tweet.status,
+                                          :created_at => tweet.created_at,
+                                          :link => tweet.public_link } }
+        File.open( TBConfig.options[:backup_file], "w" ) { |f| YAML::dump( tweets, f ) }
+      end
+    end
+
+    def public_link
+      "https://twitter.com/#{TBConfig.user.screen_name}/status/#{self.status_id}"
+    end
+
+  end
+end

data/lib/twitter_backup/ui.rb

@@ -0,0 +1,89 @@
+module TwitterBackup
+  module UI
+    class << self
+      def ask_credentials
+        say "You need to give us necessary credentials"
+        say "Get them at https://dev.twitter.com/apps"
+
+        existing_credentials = TBConfig.options[:credentials]
+        new_credentials = {}
+
+        new_credentials[:consumer_key]       = ask("Consumer key?  ").to_s          if existing_credentials[:consumer_key].blank?
+        new_credentials[:consumer_secret]    = ask("Consumer secret?  ").to_s       if existing_credentials[:consumer_secret].blank?
+        new_credentials[:oauth_token]        = ask("Access token?  ").to_s          if existing_credentials[:oauth_token].blank?
+        new_credentials[:oauth_token_secret] = ask("Access token secret?  ").to_s   if existing_credentials[:oauth_token_secret].blank?
+
+        TBConfig.save_credentials new_credentials
+        raise MissingCredentials if TBConfig.credentials_missing?
+      end
+
+      def define_path_to_backup
+        default_path = File.join(CONFIG_DIR, "tweets.yml")
+        say "We are going to save your tweets to #{TBConfig.options[:db][:database]}"
+        say "And also as plain text to #{default_path}"
+        if agree ("Dou you want to define another directory for text copy?  ")
+          backup_dir = ask("Enter a path to your direcory:  ") do |q|
+            q.validate = lambda { |file| File.exist?(File.dirname(File.expand_path(file))) }
+            q.responses[:not_valid] = "You can't save files there"
+            q.confirm  = true
+          end
+          backup_file = File.expand_path(File.join(backup_dir, "tweets.yml"))
+        else
+          backup_file = default_path
+        end
+        
+        TBConfig.save_backup_file backup_file
+        
+      end
+
+      def greet_user
+        say %[.... <%= color("Tweets for: #{TBConfig.user.name}", GREEN) %>!]
+        say %[.... Your tweets at twitter.com: #{TBConfig.user.statuses_count}]
+      end
+
+      def exit_screen
+        if ActiveRecord::Base.connected?
+          say %[.... Your tweets in this backup: #{Tweet.count}]
+          if Tweet.earliest.present?
+            say %[.... Your earliest tweet:]
+            say %[.... <%= color("#{Tweet.earliest.try(:status)}", YELLOW) %>]
+          end
+          if Tweet.latest.present?
+            say %[.... Your latest tweet:]
+            say %[.... <%= color("#{Tweet.latest.try(:status)}", GREEN) %>]
+          end
+        end
+      end
+
+      def missing_credentials_exit
+        say %[.... <%= color("We can't work without credentials. Sorry.", RED) %>]
+        say ".... Go get them at https://dev.twitter.com/apps"
+        say ".... And run this script again"
+        exit 1
+      end
+
+      def wrong_credentials_exit
+        say %[.... <%= color("Your credentials are somehow wrong.", RED) %>]
+        say ".... Go get the right ones at https://dev.twitter.com/apps"
+        say ".... And run this script again"
+        exit 1
+      end
+
+      def failed_backup_path_exit message
+        say %[.... <%= color("We've failed to create #{TBConfig.options[:backup_file]} for you.", RED) %>]
+        say %[.... #{message}]
+        TBConfig.save_backup_file("")
+        say ".... Run this script again and chose another place to store your backup"
+        say ".... Or edit it manually at #{TBConfig.config_file}"
+        exit 1
+      end
+
+      def too_many_requests_exit
+        say %[.... <%= color("You've exceeded your request limit", RED) %>]
+        say ".... Try again tomorrow =)"
+        exit 1
+      end
+
+    end
+  end
+end

data/lib/twitter_backup/version.rb

@@ -0,0 +1,3 @@
+module TwitterBackup
+  VERSION = "0.1.0"
+end

data/twitter_backup.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/twitter_backup/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["il.zoff"]
+  gem.email         = ["il.zoff@gmail.com"]
+  gem.description   = %q{This gem will download your tweets from Twitter and save them to sqlite3 database and plaintext (yaml) archive file.}
+  gem.summary       = %q{Twitter archiver}
+  gem.homepage      = "http://github.com/ilzoff/twitter_backup"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "twitter_backup"
+  gem.require_paths = ["lib"]
+  gem.version       = TwitterBackup::VERSION
+
+  gem.add_dependency("highline")
+  gem.add_dependency("activesupport")
+  gem.add_dependency("activerecord")
+  gem.add_dependency("sqlite3")
+  gem.add_dependency("twitter")
+  gem.add_dependency("slop")
+
+end

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: twitter_backup
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- il.zoff
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem will download your tweets from Twitter and save them to sqlite3
+  database and plaintext (yaml) archive file.
+email:
+- il.zoff@gmail.com
+executables:
+- twitter_backup
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/twitter_backup
+- lib/twitter_backup.rb
+- lib/twitter_backup/config.rb
+- lib/twitter_backup/error.rb
+- lib/twitter_backup/tweet.rb
+- lib/twitter_backup/ui.rb
+- lib/twitter_backup/version.rb
+- twitter_backup.gemspec
+homepage: http://github.com/ilzoff/twitter_backup
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Twitter archiver
+test_files: []