google_plus_archiver 0.0.1

data/bin/gplus-get

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'google_plus_archiver'
+require 'google_plus_archiver/version.rb'
+
+require 'optparse'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: gplus-get -a [API_KEY] -u [USER_ID]"
+  
+  opts.on("--api-key [API_KEY]", "Specify the Google API key") do |api_key|
+    options[:api_key] = api_key
+  end
+  
+  opts.on("--user-id [USER_ID]" ,"Specify the ID of the user to be archived") do |user_id|
+    options[:user_id] = user_id
+  end
+  
+  opts.on("--delay [SECONDS]" ,"Delay (in seconds) between two requests (0.2 by default, since Google set a 5 requests/second/user limit)") do |delay|
+    options[:delay] = delay
+  end
+  
+  opts.on("--output-path [OUTPUT_PATH]" ,"Output path (the current directory by default)") do |output_path|
+    options[:output_path] = output_path
+  end
+  
+  opts.on("--quiet", "Silent mode") do
+    options[:quiet] = true
+  end
+  
+  opts.on("--exclude-posts", "Don't archive posts") do
+    options[:exclude_posts] = true
+  end
+  
+  opts.on("--exclude-attachments", "Don't archive attachments") do
+    options[:exclude_attachments] = true
+  end
+  
+  opts.on("--exclude-replies", "Don't archive replies") do
+    options[:exclude_replies] = true
+  end
+  
+  opts.on("--exclude-plusoners", "Don't archive plusoners") do
+    options[:exclude_plusoners] = true
+  end
+  
+  opts.on("--exclude-resharers", "Don't archive resharers") do
+    options[:exclude_resharers] = true
+  end
+  
+  opts.on("--version", "Display current version") do
+    puts "google_plus_archiver #{GooglePlusArchiver::VERSION}"
+    exit 0
+  end
+  
+end.parse!
+
+if not options[:api_key] or not options[:user_id]
+  puts "You must specify both the user ID (-u) and your Google API key (-a)."
+  exit 0
+end
+
+GooglePlusArchiver::register_client(options[:api_key])
+if GooglePlusArchiver::client_registered?
+  GooglePlusArchiver::archive_user(options)
+end

data/lib/google_plus_archiver/version.rb

@@ -0,0 +1,4 @@
+module GooglePlusArchiver
+  VERSION       = "0.0.1"
+  DATE          = "2012-12-18"
+end

data/lib/google_plus_archiver.rb

@@ -0,0 +1,282 @@
+require 'json'
+require 'net/http'
+require 'tempfile'
+require 'tmpdir'
+require 'zlib'
+
+require 'google/api_client'
+
+require 'archive/tar/minitar'
+include Archive::Tar
+
+module GooglePlusArchiver
+  
+  def self.api_key
+    @@api_key
+  end
+  
+  def self.api_key=(api_key)
+    @@api_key = @@client.key = api_key
+  end
+  
+  def self.request_num
+    @@request_num
+  end
+  
+  def self.register_client(api_key)
+    @@client = Google::APIClient.new
+    @@api_key = @@client.key = api_key
+    @@request_num = 0
+    begin
+      @@plus = @@client.discovered_api('plus')
+    rescue
+      puts "Invalid Google API key."
+    end
+  end
+  
+  def self.client_registered?
+    defined? @@plus
+  end
+  
+  def self.archive_user(params)
+    begin
+      raise "Unregistered client." unless client_registered?
+    rescue => e
+      puts e.message
+      return
+    end
+    
+    user_id, delay, output_path, quiet =
+      (params[:user_id]),
+      (params[:delay] or 0.2),
+      (params[:output_path] or FileUtils.pwd),
+      (params[:quiet])
+    
+    Dir.mktmpdir do |tmp_dir|
+      
+      begin
+        
+        #>> profile
+        puts "##{@@request_num+=1} Fetching people.get ..." unless quiet
+        response = @@client.execute(
+          :api_method => @@plus.people.get,
+          :parameters => {
+            'collection' => 'public',
+            'userId' => user_id
+          },
+          :authenticated => false
+        )
+        
+        #<< profile
+        File.open("#{tmp_dir}/profile.json", "w") do |f|
+          f.puts response.body
+        end
+        
+        user_display_name = JSON.parse(response.body)['displayName']
+        
+        #>> posts
+        if not params[:exclude_posts]
+          next_page_token = nil
+          page_num = 0
+          loop do
+            puts "##{@@request_num+=1} Fetching activities.list: page[#{page_num}] ..." unless quiet
+            response = @@client.execute(
+              :api_method => @@plus.activities.list,
+              :parameters => {
+                'collection' => 'public',
+                'userId' => user_id,
+                'maxResults' => '100',
+                'pageToken' => next_page_token
+              },
+              :authenticated => false
+            )
+            activities = JSON.parse(response.body)
+            next_page_token = activities['nextPageToken']
+            
+            #<< posts
+            File.open("#{tmp_dir}/posts[#{page_num}].json", "w") do |f|
+              f.puts response.body
+            end
+            
+            activities['items'].each do |item|
+              activity_id = item['id']
+              
+              puts "##{@@request_num}   Fetching activities.get: #{activity_id}" unless quiet
+              
+              #<< post
+              File.open("#{tmp_dir}/#{activity_id}.json", "w") do |f|
+                f.puts item.to_json
+              end
+              
+              #>> attachments
+              if not params[:exclude_attachments] and item['object']['attachments']
+                item['object']['attachments'].each do |attachment|
+                  image = (attachment['fullImage'] or attachment['image'])
+                  if image
+                    puts "##{@@request_num}     Fetching attachment: #{image['url']} ..." unless quiet
+                    uri = URI.parse(URI.escape("#{image['url']}"))
+                    http = Net::HTTP.new(uri.host, uri.port)
+                    if http.port == 443
+                      http.use_ssl = true
+                      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+                    end
+                    data = http.get(uri.request_uri)
+                    image_ext = uri.request_uri.split("/")[-1].split(".")[-1]
+                    image_ext = nil if image_ext.length > 4
+                    
+                    #<< attachment
+                    File.open("#{tmp_dir}/#{activity_id}_#{attachment['id']}#{image_ext ? ".#{image_ext}" : ""}", "w").puts data.body
+                  end
+                  
+                  thumbnails = attachment['thumbnails']
+                  if thumbnails
+                    thumbnails.each_index do |index|
+                      thumbnail = thumbnails[index]
+                      image = thumbnail['image']
+                      puts "##{@@request_num}     Fetching attachment(thumbnail): #{image['url']} ..." unless quiet
+                      uri = URI.parse(URI.escape("#{image['url']}"))
+                      http = Net::HTTP.new(uri.host, uri.port)
+                      if http.port == 443
+                        http.use_ssl = true
+                        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+                      end
+                      data = http.get(uri.request_uri)
+                      image_ext = uri.request_uri.split("/")[-1].split(".")[-1]
+                      image_ext = nil if image_ext.length > 4
+                      
+                      #<< attachment
+                      File.open("#{tmp_dir}/#{activity_id}_#{attachment['id']}_#{index.to_s}#{image_ext ? ".#{image_ext}" : ""}", "w").puts data.body
+                    end
+                  end
+                end
+              end
+              
+              #>> replies
+              if not params[:exclude_replies]
+                replies_next_page_token = nil
+                replies_page_num = 0
+                loop do
+                  puts "##{@@request_num+=1}     Fetching comments.list: page[#{replies_page_num}] ..." unless quiet
+                  response = @@client.execute(
+                    :api_method => @@plus.comments.list,
+                    :parameters => {
+                      'activityId' => activity_id,
+                      'maxResults' => '500',
+                      'pageToken' => replies_next_page_token
+                    },
+                    :authenticated => false
+                  )
+                  replies_next_page_token = JSON.parse(response.body)['nextPageToken']
+                  
+                  #<< replies
+                  File.open("#{tmp_dir}/#{activity_id}_replies#{replies_page_num == 0 && !replies_next_page_token ? "" : "[#{replies_page_num}]"}.json", "w") do |f|
+                    f.puts response.body
+                  end
+                  
+                  break unless replies_next_page_token
+                  replies_page_num += 1
+                  sleep delay
+                end
+              end
+              
+              #>> plusoners
+              if not params[:exclude_plusoners]
+                plusoners_next_page_token = nil
+                plusoners_page_num = 0
+                loop do
+                  puts "##{@@request_num+=1}     Fetching people.listByActivity(plusoners): page[#{plusoners_page_num}] ..." unless quiet
+                  response = @@client.execute(
+                    :api_method => @@plus.people.list_by_activity,
+                    :parameters => {
+                      'activityId' => activity_id,
+                      'collection' => 'plusoners',
+                      'maxResults' => '100',
+                      'pageToken' => plusoners_next_page_token
+                    },
+                    :authenticated => false
+                  )
+                  plusoners_next_page_token = JSON.parse(response.body)['nextPageToken']
+                  
+                  #<< plusoners
+                  File.open("#{tmp_dir}/#{activity_id}_plusoners#{plusoners_page_num == 0 && !plusoners_next_page_token ? "" : "[#{plusoners_page_num}]"}.json", "w") do |f|
+                    f.puts response.body
+                  end
+                  
+                  break unless plusoners_next_page_token
+                  plusoners_page_num += 1
+                  sleep delay
+                end
+              end
+              
+              #>> resharers
+              if not params[:exclude_resharers]
+                resharers_next_page_token = nil
+                resharers_page_num = 0
+                loop do
+                  puts "##{@@request_num+=1}     Fetching people.listByActivity(resharers): page[#{resharers_page_num}] ..." unless quiet
+                  response = @@client.execute(
+                    :api_method => @@plus.people.list_by_activity,
+                    :parameters => {
+                      'activityId' => activity_id,
+                      'collection' => 'resharers',
+                      'maxResults' => '100',
+                      'pageToken' => resharers_next_page_token
+                    },
+                    :authenticated => false
+                  )
+                  resharers_next_page_token = JSON.parse(response.body)['nextPageToken']
+                  
+                  #<< resharers
+                  File.open("#{tmp_dir}/#{activity_id}_resharers#{replies_page_num == 0 && !resharers_next_page_token ? "" : "[#{resharers_page_num}]"}.json", "w") do |f|
+                    f.puts response.body
+                  end
+                  
+                  break unless resharers_next_page_token
+                  resharers_page_num += 1
+                  sleep delay
+                end
+              end
+              
+            end
+            
+            break unless next_page_token
+            page_num += 1
+            sleep delay
+          end
+          
+        end
+        
+      rescue Exception => e
+        puts e.message
+        puts "Archiving interrupted due to unexpected errors."
+        
+      ensure
+        # Archive all the files
+        archive_time = "#{Time.now.to_s[0..9]}-#{Time.now.to_s[11..-7]}#{Time.now.to_s[-5..-1]}"
+        archive_filename = "#{output_path}/#{user_display_name}_#{archive_time}.tar.gz"
+        FileUtils.cd(tmp_dir) do
+          
+          Tempfile.open("#{user_id}") do |tar|
+            files = []
+            Find.find("./") do |path|
+              files << File.basename(path) unless File.basename(path) == '.'
+            end
+            Minitar.pack(files, tar)
+            
+            Zlib::GzipWriter.open(archive_filename) do |gz|
+              gz.mtime = File.mtime(tar.path)
+              gz.orig_name = tar.path
+              gz.write IO.binread(tar.path)
+            end
+            
+          end
+          
+        end
+        
+      end
+      
+    end
+    
+  end
+  
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: google_plus_archiver
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Mort Yao
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: archive-tar-minitar
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+description: google_plus_archiver is a simple command-line tool to archive Google+
+  profiles and public streams.
+email: mort.yao@gmail.com
+executables:
+- gplus-get
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/gplus-get
+- lib/google_plus_archiver.rb
+- lib/google_plus_archiver/version.rb
+homepage: https://github.com/soimort/google_plus_archiver
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: A simple command-line tool to archive Google+ profiles.
+test_files: []