google_plus_archiver 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/gplus-get +23 -6
- data/bin/gplus-image-get +64 -0
- data/lib/google_plus_archiver.rb +312 -99
- data/lib/google_plus_archiver/version.rb +2 -2
- metadata +9 -13
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7144b6b00ee7bdc554de12881d10de74c6630938
|
4
|
+
data.tar.gz: 10294ebca4f4512e0eaf10f49e08dd78dfd43f55
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 39de1a8368fe5e24026de17919b0d54befea6834248549a50e5436beafaa681be9b0a6a9ff8555e809ab27d1ab8724c27eab7487f44b9f988e349a407cc696b4
|
7
|
+
data.tar.gz: 0137ff5c4d3a6fdf96b0116160d0f0978b84acf3e06631ae7bc8662324c130edf7713728d5408b7817554b55b9f98457f756530cbdc184efd57e8ffdfa7b558b
|
data/bin/gplus-get
CHANGED
@@ -15,22 +15,34 @@ OptionParser.new do |opts|
|
|
15
15
|
options[:api_key] = api_key
|
16
16
|
end
|
17
17
|
|
18
|
-
opts.on("--user-id [USER_ID]"
|
18
|
+
opts.on("--user-id [USER_ID]", "Specify the ID of the user to be archived") do |user_id|
|
19
19
|
options[:user_id] = user_id
|
20
20
|
end
|
21
21
|
|
22
|
-
opts.on("--
|
22
|
+
opts.on("--compress", "Do compression") do
|
23
|
+
options[:compress] = true
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on("--delay [SECONDS]", "Delay (in seconds) between two requests (0.2 by default, since Google set a 5 requests/second/user limit)") do |delay|
|
23
27
|
options[:delay] = delay
|
24
28
|
end
|
25
29
|
|
26
|
-
opts.on("--output-path [OUTPUT_PATH]"
|
30
|
+
opts.on("--output-path [OUTPUT_PATH]", "Output path (the current directory by default)") do |output_path|
|
27
31
|
options[:output_path] = output_path
|
28
32
|
end
|
29
33
|
|
34
|
+
opts.on("--post-limit [POST_LIMIT]", "Maximum number of posts to archive (in time descending order)") do |post_limit|
|
35
|
+
options[:post_limit] = post_limit
|
36
|
+
end
|
37
|
+
|
30
38
|
opts.on("--quiet", "Silent mode") do
|
31
39
|
options[:quiet] = true
|
32
40
|
end
|
33
41
|
|
42
|
+
opts.on("--video-downloader [VIDEO_DOWNLOADER]", "Command used to download Google+ videos (`you-get` by default)") do |video_downloader|
|
43
|
+
options[:video_downloader] = video_downloader
|
44
|
+
end
|
45
|
+
|
34
46
|
opts.on("--exclude-posts", "Don't archive posts") do
|
35
47
|
options[:exclude_posts] = true
|
36
48
|
end
|
@@ -51,19 +63,24 @@ OptionParser.new do |opts|
|
|
51
63
|
options[:exclude_resharers] = true
|
52
64
|
end
|
53
65
|
|
54
|
-
opts.on("--version", "Display current version") do
|
66
|
+
opts.on("-V", "--version", "Display current version") do
|
55
67
|
puts "google_plus_archiver #{GooglePlusArchiver::VERSION}"
|
56
68
|
exit 0
|
57
69
|
end
|
58
70
|
|
59
71
|
end.parse!
|
60
72
|
|
73
|
+
options[:api_key] = ENV['GOOGLE_API_KEY'] if not options[:api_key]
|
74
|
+
|
61
75
|
if not options[:api_key] or not options[:user_id]
|
62
76
|
puts "You must specify both the user ID (-u) and your Google API key (-a)."
|
63
77
|
exit 0
|
64
78
|
end
|
65
79
|
|
66
80
|
GooglePlusArchiver::register_client(options[:api_key])
|
67
|
-
if GooglePlusArchiver::client_registered?
|
68
|
-
|
81
|
+
if not GooglePlusArchiver::client_registered?
|
82
|
+
puts "Client registration failed."
|
83
|
+
exit 0
|
69
84
|
end
|
85
|
+
|
86
|
+
GooglePlusArchiver::archive_user(options)
|
data/bin/gplus-image-get
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
|
4
|
+
|
5
|
+
require 'google_plus_archiver'
|
6
|
+
require 'google_plus_archiver/version.rb'
|
7
|
+
|
8
|
+
require 'optparse'
|
9
|
+
require 'net/http'
|
10
|
+
|
11
|
+
options = {}
|
12
|
+
OptionParser.new do |opts|
|
13
|
+
opts.banner = "Usage: gplus-image-get -a [API_KEY] [POST_URL]"
|
14
|
+
|
15
|
+
opts.on("--api-key [API_KEY]", "Specify the Google API key") do |api_key|
|
16
|
+
options[:api_key] = api_key
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.on("--output-path [OUTPUT_PATH]", "Output path (the current directory by default)") do |output_path|
|
20
|
+
options[:output_path] = output_path
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on("-x", "--proxy [PROXY]", "Use proxy on given port") do |proxy|
|
24
|
+
options[:proxy] = proxy
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on("-V", "--version", "Display current version") do
|
28
|
+
puts "google_plus_archiver #{GooglePlusArchiver::VERSION}"
|
29
|
+
exit 0
|
30
|
+
end
|
31
|
+
|
32
|
+
end.parse!
|
33
|
+
|
34
|
+
options[:api_key] = ENV['GOOGLE_API_KEY'] if not options[:api_key]
|
35
|
+
|
36
|
+
if not options[:api_key] or ARGV.empty?
|
37
|
+
puts "You must specify both the post URL and your Google API key (-a)."
|
38
|
+
exit 0
|
39
|
+
end
|
40
|
+
|
41
|
+
proxy_addr, proxy_port = options[:proxy].split(':') if options[:proxy]
|
42
|
+
|
43
|
+
GooglePlusArchiver::register_client(options[:api_key])
|
44
|
+
if not GooglePlusArchiver::client_registered?
|
45
|
+
puts "Client registration failed."
|
46
|
+
exit 0
|
47
|
+
end
|
48
|
+
|
49
|
+
ARGV.each do |url|
|
50
|
+
uri = URI.parse(URI.escape("#{url}"))
|
51
|
+
if options[:proxy]
|
52
|
+
http = Net::HTTP.new(uri.host, uri.port, proxy_addr, proxy_port)
|
53
|
+
else
|
54
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
55
|
+
end
|
56
|
+
if http.port == 443
|
57
|
+
http.use_ssl = true
|
58
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
59
|
+
end
|
60
|
+
data = http.get(uri.request_uri)
|
61
|
+
|
62
|
+
activity_id = data.body.match(/<div id="update-([^"]+)"/)[1]
|
63
|
+
GooglePlusArchiver::fetch_post_image(:activity_id => activity_id, :output_path => options[:output_path])
|
64
|
+
end
|
data/lib/google_plus_archiver.rb
CHANGED
@@ -24,7 +24,7 @@ module GooglePlusArchiver
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def self.register_client(api_key)
|
27
|
-
@@client = Google::APIClient.new
|
27
|
+
@@client = Google::APIClient.new(:application_name => 'GooglePlusArchiver', :application_version => VERSION)
|
28
28
|
@@api_key = @@client.key = api_key
|
29
29
|
@@request_num = 0
|
30
30
|
begin
|
@@ -38,6 +38,20 @@ module GooglePlusArchiver
|
|
38
38
|
defined? @@plus
|
39
39
|
end
|
40
40
|
|
41
|
+
def self.get_full_image_url(url)
|
42
|
+
if url =~ /https:\/\/\w+\.googleusercontent\.com/
|
43
|
+
if url =~ /\/s\d+\/[^\/]+$/ or url =~ /\/w\d+-h\d+\/[^\/]+$/ or url =~ /\/w\d+-h\d+-\w+\/[^\/]+$/
|
44
|
+
url[0..url[0..(url.rindex('/') - 1)].rindex('/')] + 's0-d' + url[url.rindex('/')..-1]
|
45
|
+
elsif url =~ /\/photo.jpg$/ and not url =~ /\/s0-d\/[^\/]+$/
|
46
|
+
url[0..url.rindex('/')] + 's0-d' + url[url.rindex('/')..-1]
|
47
|
+
else
|
48
|
+
url
|
49
|
+
end
|
50
|
+
else
|
51
|
+
url
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
41
55
|
def self.archive_user(params)
|
42
56
|
begin
|
43
57
|
raise "Unregistered client." unless client_registered?
|
@@ -46,29 +60,41 @@ module GooglePlusArchiver
|
|
46
60
|
return
|
47
61
|
end
|
48
62
|
|
49
|
-
user_id, delay, output_path, quiet =
|
63
|
+
user_id, compress, delay, output_path, post_limit, quiet, video_downloader =
|
50
64
|
(params[:user_id]),
|
65
|
+
(params[:compress]),
|
51
66
|
(params[:delay] or 0.2),
|
52
67
|
(params[:output_path] or FileUtils.pwd),
|
53
|
-
(params[:
|
68
|
+
(params[:post_limit]),
|
69
|
+
(params[:quiet]),
|
70
|
+
(params[:video_downloader] or 'you-get')
|
54
71
|
|
55
72
|
Dir.mktmpdir do |tmp_dir|
|
56
|
-
|
57
73
|
begin
|
74
|
+
response = nil
|
58
75
|
|
59
76
|
#>> profile
|
60
77
|
puts "##{@@request_num+=1} Fetching people.get ..." unless quiet
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
78
|
+
loop do
|
79
|
+
begin
|
80
|
+
response = @@client.execute(
|
81
|
+
:api_method => @@plus.people.get,
|
82
|
+
:parameters => {
|
83
|
+
'collection' => 'public',
|
84
|
+
'userId' => user_id
|
85
|
+
},
|
86
|
+
:authenticated => false
|
87
|
+
)
|
88
|
+
rescue
|
89
|
+
puts "##{@@request_num} Retrying people.get ..." unless quiet
|
90
|
+
next
|
91
|
+
else
|
92
|
+
break
|
93
|
+
end
|
94
|
+
end
|
69
95
|
|
70
96
|
#<< profile
|
71
|
-
File.open("#{tmp_dir
|
97
|
+
File.open("#{File.join(tmp_dir, 'profile.json')}", "w") do |f|
|
72
98
|
f.puts response.body
|
73
99
|
end
|
74
100
|
|
@@ -78,23 +104,41 @@ module GooglePlusArchiver
|
|
78
104
|
if not params[:exclude_posts]
|
79
105
|
next_page_token = nil
|
80
106
|
page_num = 0
|
107
|
+
posts_left = post_limit.to_i
|
108
|
+
|
81
109
|
loop do
|
82
110
|
puts "##{@@request_num+=1} Fetching activities.list: page[#{page_num}] ..." unless quiet
|
83
|
-
|
84
|
-
:
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
111
|
+
if post_limit
|
112
|
+
maxResults = (posts_left > 100) ? 100 : posts_left
|
113
|
+
posts_left -= maxResults
|
114
|
+
else
|
115
|
+
maxResults = 100
|
116
|
+
end
|
117
|
+
loop do
|
118
|
+
begin
|
119
|
+
response = @@client.execute(
|
120
|
+
:api_method => @@plus.activities.list,
|
121
|
+
:parameters => {
|
122
|
+
'collection' => 'public',
|
123
|
+
'userId' => user_id,
|
124
|
+
'maxResults' => maxResults.to_s,
|
125
|
+
'pageToken' => next_page_token
|
126
|
+
},
|
127
|
+
:authenticated => false
|
128
|
+
)
|
129
|
+
rescue
|
130
|
+
puts "##{@@request_num} Retrying activities.list: page[#{page_num}] ..." unless quiet
|
131
|
+
next
|
132
|
+
else
|
133
|
+
break
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
93
137
|
activities = JSON.parse(response.body)
|
94
138
|
next_page_token = activities['nextPageToken']
|
95
139
|
|
96
140
|
#<< posts
|
97
|
-
File.open("#{tmp_dir}
|
141
|
+
File.open("#{File.join(tmp_dir, 'posts')}[#{page_num}].json", "w") do |f|
|
98
142
|
f.puts response.body
|
99
143
|
end
|
100
144
|
|
@@ -104,37 +148,20 @@ module GooglePlusArchiver
|
|
104
148
|
puts "##{@@request_num} Fetching activities.get: #{activity_id}" unless quiet
|
105
149
|
|
106
150
|
#<< post
|
107
|
-
File.open("#{tmp_dir
|
151
|
+
File.open("#{File.join(tmp_dir, activity_id)}.json", "w") do |f|
|
108
152
|
f.puts item.to_json
|
109
153
|
end
|
110
154
|
|
111
155
|
#>> attachments
|
112
156
|
if not params[:exclude_attachments] and item['object']['attachments']
|
113
157
|
item['object']['attachments'].each do |attachment|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
122
|
-
end
|
123
|
-
data = http.get(uri.request_uri)
|
124
|
-
image_ext = uri.request_uri.split("/")[-1].split(".")[-1]
|
125
|
-
image_ext = nil if image_ext.length > 4
|
126
|
-
|
127
|
-
#<< attachment
|
128
|
-
File.open("#{tmp_dir}/#{activity_id}_#{attachment['id']}#{image_ext ? ".#{image_ext}" : ""}", "w").puts data.body
|
129
|
-
end
|
130
|
-
|
131
|
-
thumbnails = attachment['thumbnails']
|
132
|
-
if thumbnails
|
133
|
-
thumbnails.each_index do |index|
|
134
|
-
thumbnail = thumbnails[index]
|
135
|
-
image = thumbnail['image']
|
136
|
-
puts "##{@@request_num} Fetching attachment(thumbnail): #{image['url']} ..." unless quiet
|
137
|
-
uri = URI.parse(URI.escape("#{image['url']}"))
|
158
|
+
if attachment['objectType'] == 'photo'
|
159
|
+
# Download full-size image
|
160
|
+
begin
|
161
|
+
image = attachment['fullImage']
|
162
|
+
image_url = get_full_image_url(image['url'])
|
163
|
+
puts "##{@@request_num} Fetching attachment: #{image_url} ..." unless quiet
|
164
|
+
uri = URI.parse(URI.escape("#{image_url}"))
|
138
165
|
http = Net::HTTP.new(uri.host, uri.port)
|
139
166
|
if http.port == 443
|
140
167
|
http.use_ssl = true
|
@@ -145,8 +172,81 @@ module GooglePlusArchiver
|
|
145
172
|
image_ext = nil if image_ext.length > 4
|
146
173
|
|
147
174
|
#<< attachment
|
148
|
-
File.open("#{tmp_dir
|
175
|
+
File.open("#{File.join(tmp_dir, activity_id)}_#{attachment['id']}#{image_ext ? ".#{image_ext}" : ""}", "w").puts data.body
|
176
|
+
rescue
|
177
|
+
image = attachment['image']
|
178
|
+
image_url = get_full_image_url(image['url'])
|
179
|
+
puts "##{@@request_num} Fetching attachment: #{image_url} ..." unless quiet
|
180
|
+
uri = URI.parse(URI.escape("#{image_url}"))
|
181
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
182
|
+
if http.port == 443
|
183
|
+
http.use_ssl = true
|
184
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
185
|
+
end
|
186
|
+
data = http.get(uri.request_uri)
|
187
|
+
image_ext = uri.request_uri.split("/")[-1].split(".")[-1]
|
188
|
+
image_ext = nil if image_ext.length > 4
|
189
|
+
|
190
|
+
#<< attachment
|
191
|
+
File.open("#{File.join(tmp_dir, activity_id)}_#{attachment['id']}#{image_ext ? ".#{image_ext}" : ""}", "w").puts data.body
|
192
|
+
end
|
193
|
+
|
194
|
+
elsif attachment['objectType'] == 'album'
|
195
|
+
# Download full-size thumbnails
|
196
|
+
thumbnails = attachment['thumbnails']
|
197
|
+
if thumbnails
|
198
|
+
thumbnails.each_index do |index|
|
199
|
+
thumbnail = thumbnails[index]
|
200
|
+
image = thumbnail['image']
|
201
|
+
image_url = get_full_image_url(image['url'])
|
202
|
+
puts "##{@@request_num} Fetching attachment: #{image_url} ..." unless quiet
|
203
|
+
uri = URI.parse(URI.escape("#{image_url}"))
|
204
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
205
|
+
if http.port == 443
|
206
|
+
http.use_ssl = true
|
207
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
208
|
+
end
|
209
|
+
data = http.get(uri.request_uri)
|
210
|
+
image_ext = uri.request_uri.split("/")[-1].split(".")[-1]
|
211
|
+
image_ext = nil if image_ext.length > 4
|
212
|
+
|
213
|
+
#<< attachment
|
214
|
+
File.open("#{File.join(tmp_dir, activity_id)}_#{attachment['id']}[#{index}]#{image_ext ? ".#{image_ext}" : ""}", "w").puts data.body
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
elsif attachment['objectType'] == 'video'
|
219
|
+
# Download preview image
|
220
|
+
image = attachment['image']
|
221
|
+
image_url = get_full_image_url(image['url'])
|
222
|
+
puts "##{@@request_num} Fetching attachment: #{image_url} ..." unless quiet
|
223
|
+
uri = URI.parse(URI.escape("#{image_url}"))
|
224
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
225
|
+
if http.port == 443
|
226
|
+
http.use_ssl = true
|
227
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
149
228
|
end
|
229
|
+
data = http.get(uri.request_uri)
|
230
|
+
image_ext = 'gif'
|
231
|
+
|
232
|
+
extname = data.header['Content-Type'].split('/')[-1]
|
233
|
+
|
234
|
+
#<< attachment
|
235
|
+
File.open("#{File.join(tmp_dir, activity_id)}_#{attachment['id']}.#{extname}", "w").puts data.body
|
236
|
+
|
237
|
+
# Download video
|
238
|
+
puts "##{@@request_num} Downloading video: #{attachment['url']} ..." unless quiet
|
239
|
+
FileUtils.mkdir("#{File.join(tmp_dir, 'video')}")
|
240
|
+
Dir.chdir("#{File.join(tmp_dir, 'video')}") do
|
241
|
+
if system("#{video_downloader} #{attachment['url']}")
|
242
|
+
Dir.glob("*").each do |video|
|
243
|
+
FileUtils.mv(video, "#{File.join(tmp_dir, activity_id)}_#{attachment['id']}_#{attachment['displayName'].split('/').join}.#{video.split('.')[-1]}")
|
244
|
+
end
|
245
|
+
else
|
246
|
+
puts "##{@@request_num} Video downloader failed. Download aborted."
|
247
|
+
end
|
248
|
+
end
|
249
|
+
FileUtils.rm_r("#{File.join(tmp_dir, 'video')}")
|
150
250
|
end
|
151
251
|
end
|
152
252
|
end
|
@@ -157,19 +257,29 @@ module GooglePlusArchiver
|
|
157
257
|
replies_page_num = 0
|
158
258
|
loop do
|
159
259
|
puts "##{@@request_num+=1} Fetching comments.list: page[#{replies_page_num}] ..." unless quiet
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
260
|
+
loop do
|
261
|
+
begin
|
262
|
+
response = @@client.execute(
|
263
|
+
:api_method => @@plus.comments.list,
|
264
|
+
:parameters => {
|
265
|
+
'activityId' => activity_id,
|
266
|
+
'maxResults' => '500',
|
267
|
+
'pageToken' => replies_next_page_token
|
268
|
+
},
|
269
|
+
:authenticated => false
|
270
|
+
)
|
271
|
+
rescue
|
272
|
+
puts "##{@@request_num} Retrying comments.list: page[#{replies_page_num}] ..." unless quiet
|
273
|
+
next
|
274
|
+
else
|
275
|
+
break
|
276
|
+
end
|
277
|
+
end
|
278
|
+
|
169
279
|
replies_next_page_token = JSON.parse(response.body)['nextPageToken']
|
170
280
|
|
171
281
|
#<< replies
|
172
|
-
File.open("#{tmp_dir
|
282
|
+
File.open("#{File.join(tmp_dir, activity_id)}_replies#{replies_page_num == 0 && !replies_next_page_token ? "" : "[#{replies_page_num}]"}.json", "w") do |f|
|
173
283
|
f.puts response.body
|
174
284
|
end
|
175
285
|
|
@@ -185,20 +295,30 @@ module GooglePlusArchiver
|
|
185
295
|
plusoners_page_num = 0
|
186
296
|
loop do
|
187
297
|
puts "##{@@request_num+=1} Fetching people.listByActivity(plusoners): page[#{plusoners_page_num}] ..." unless quiet
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
298
|
+
loop do
|
299
|
+
begin
|
300
|
+
response = @@client.execute(
|
301
|
+
:api_method => @@plus.people.list_by_activity,
|
302
|
+
:parameters => {
|
303
|
+
'activityId' => activity_id,
|
304
|
+
'collection' => 'plusoners',
|
305
|
+
'maxResults' => '100',
|
306
|
+
'pageToken' => plusoners_next_page_token
|
307
|
+
},
|
308
|
+
:authenticated => false
|
309
|
+
)
|
310
|
+
rescue
|
311
|
+
puts "##{@@request_num} Retrying people.listByActivity(plusoners): page[#{plusoners_page_num}] ..." unless quiet
|
312
|
+
next
|
313
|
+
else
|
314
|
+
break
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
198
318
|
plusoners_next_page_token = JSON.parse(response.body)['nextPageToken']
|
199
319
|
|
200
320
|
#<< plusoners
|
201
|
-
File.open("#{tmp_dir
|
321
|
+
File.open("#{File.join(tmp_dir, activity_id)}_plusoners#{plusoners_page_num == 0 && !plusoners_next_page_token ? "" : "[#{plusoners_page_num}]"}.json", "w") do |f|
|
202
322
|
f.puts response.body
|
203
323
|
end
|
204
324
|
|
@@ -214,20 +334,30 @@ module GooglePlusArchiver
|
|
214
334
|
resharers_page_num = 0
|
215
335
|
loop do
|
216
336
|
puts "##{@@request_num+=1} Fetching people.listByActivity(resharers): page[#{resharers_page_num}] ..." unless quiet
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
337
|
+
loop do
|
338
|
+
begin
|
339
|
+
response = @@client.execute(
|
340
|
+
:api_method => @@plus.people.list_by_activity,
|
341
|
+
:parameters => {
|
342
|
+
'activityId' => activity_id,
|
343
|
+
'collection' => 'resharers',
|
344
|
+
'maxResults' => '100',
|
345
|
+
'pageToken' => resharers_next_page_token
|
346
|
+
},
|
347
|
+
:authenticated => false
|
348
|
+
)
|
349
|
+
rescue
|
350
|
+
puts "##{@@request_num} Retrying people.listByActivity(resharers): page[#{resharers_page_num}] ..." unless quiet
|
351
|
+
next
|
352
|
+
else
|
353
|
+
break
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
227
357
|
resharers_next_page_token = JSON.parse(response.body)['nextPageToken']
|
228
358
|
|
229
359
|
#<< resharers
|
230
|
-
File.open("#{tmp_dir
|
360
|
+
File.open("#{File.join(tmp_dir, activity_id)}_resharers#{replies_page_num == 0 && !resharers_next_page_token ? "" : "[#{resharers_page_num}]"}.json", "w") do |f|
|
231
361
|
f.puts response.body
|
232
362
|
end
|
233
363
|
|
@@ -239,6 +369,8 @@ module GooglePlusArchiver
|
|
239
369
|
|
240
370
|
end
|
241
371
|
|
372
|
+
break if post_limit and posts_left <= 0
|
373
|
+
|
242
374
|
break unless next_page_token
|
243
375
|
page_num += 1
|
244
376
|
sleep delay
|
@@ -251,32 +383,113 @@ module GooglePlusArchiver
|
|
251
383
|
puts "Archiving interrupted due to unexpected errors."
|
252
384
|
|
253
385
|
ensure
|
254
|
-
# Archive all the files
|
255
386
|
archive_time = "#{Time.now.to_s[0..9]}-#{Time.now.to_s[11..-7]}#{Time.now.to_s[-5..-1]}"
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
387
|
+
archive_dest = "#{File.join(output_path, user_display_name)}_#{archive_time}"
|
388
|
+
|
389
|
+
FileUtils.mkdir_p(archive_dest)
|
390
|
+
FileUtils.cp_r("#{File.join(tmp_dir, '.')}", archive_dest)
|
391
|
+
|
392
|
+
if compress
|
393
|
+
begin
|
394
|
+
archive_filename = "#{File.join(output_path, user_display_name)}_#{archive_time}.tar.gz"
|
395
|
+
FileUtils.cd(archive_dest) do
|
396
|
+
Tempfile.open("#{user_id}") do |tar|
|
397
|
+
files = []
|
398
|
+
Find.find("./") do |path|
|
399
|
+
files << File.basename(path) unless File.basename(path) == '.'
|
400
|
+
end
|
401
|
+
Minitar.pack(files, tar)
|
402
|
+
|
403
|
+
Zlib::GzipWriter.open(archive_filename) do |gz|
|
404
|
+
gz.mtime = File.mtime(tar.path)
|
405
|
+
gz.orig_name = tar.path
|
406
|
+
gz.write IO.binread(tar.path)
|
407
|
+
end
|
408
|
+
end
|
263
409
|
end
|
264
|
-
Minitar.pack(files, tar)
|
265
410
|
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
411
|
+
FileUtils.rm_r(archive_dest)
|
412
|
+
rescue Exception => e
|
413
|
+
puts e.message
|
414
|
+
puts "Compression failed."
|
415
|
+
end
|
416
|
+
end
|
417
|
+
end
|
418
|
+
end
|
419
|
+
end
|
420
|
+
|
421
|
+
def self.fetch_post_image(params)
|
422
|
+
begin
|
423
|
+
raise "Unregistered client." unless client_registered?
|
424
|
+
rescue => e
|
425
|
+
puts e.message
|
426
|
+
return
|
427
|
+
end
|
428
|
+
|
429
|
+
activity_id, output_path =
|
430
|
+
(params[:activity_id]),
|
431
|
+
(params[:output_path] or FileUtils.pwd)
|
432
|
+
|
433
|
+
response = @@client.execute(
|
434
|
+
:api_method => @@plus.activities.get,
|
435
|
+
:parameters => {
|
436
|
+
'activityId' => activity_id,
|
437
|
+
'fields' => 'object/attachments'
|
438
|
+
},
|
439
|
+
:authenticated => false
|
440
|
+
)
|
441
|
+
|
442
|
+
attachments = JSON.parse(response.body)['object']['attachments']
|
443
|
+
attachments.each do |attachment|
|
444
|
+
if attachment['objectType'] == 'photo'
|
445
|
+
image = attachment['fullImage']
|
446
|
+
image_url = get_full_image_url(image['url'])
|
447
|
+
puts "Downloading image: #{image_url} ..."
|
448
|
+
uri = URI.parse(URI.escape("#{image_url}"))
|
449
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
450
|
+
if http.port == 443
|
451
|
+
http.use_ssl = true
|
452
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
453
|
+
end
|
454
|
+
data = http.get(uri.request_uri)
|
455
|
+
|
456
|
+
m = data.header['Content-Disposition'].match(/filename="([^"]+)"/)
|
457
|
+
if m
|
458
|
+
extname = m[1]
|
459
|
+
else
|
460
|
+
extname = data.header['Content-Type'].split('/')[-1]
|
461
|
+
end
|
462
|
+
|
463
|
+
File.open("#{File.join(output_path, activity_id)}.#{extname}", "w").puts data.body
|
464
|
+
|
465
|
+
elsif attachment['objectType'] == 'album'
|
466
|
+
thumbnails = attachment['thumbnails']
|
467
|
+
if thumbnails
|
468
|
+
thumbnails.each_index do |index|
|
469
|
+
thumbnail = thumbnails[index]
|
470
|
+
image = thumbnail['image']
|
471
|
+
image_url = get_full_image_url(image['url'])
|
472
|
+
puts "Downloading image: #{image_url} ..."
|
473
|
+
uri = URI.parse(URI.escape("#{image_url}"))
|
474
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
475
|
+
if http.port == 443
|
476
|
+
http.use_ssl = true
|
477
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
478
|
+
end
|
479
|
+
data = http.get(uri.request_uri)
|
480
|
+
|
481
|
+
m = data.header['Content-Disposition'].match(/filename="([^"]+)"/)
|
482
|
+
if m
|
483
|
+
extname = m[1]
|
484
|
+
else
|
485
|
+
extname = data.header['Content-Type'].split('/')[-1]
|
270
486
|
end
|
271
487
|
|
488
|
+
File.open("#{File.join(output_path, activity_id)}[#{index}].#{extname}", "w").puts data.body
|
272
489
|
end
|
273
|
-
|
274
490
|
end
|
275
|
-
|
276
491
|
end
|
277
|
-
|
278
492
|
end
|
279
|
-
|
280
493
|
end
|
281
494
|
|
282
495
|
end
|
metadata
CHANGED
@@ -1,20 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google_plus_archiver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.2
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Mort Yao
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2013-07-10 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: google-api-client
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
17
|
- - ~>
|
20
18
|
- !ruby/object:Gem::Version
|
@@ -22,7 +20,6 @@ dependencies:
|
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
24
|
- - ~>
|
28
25
|
- !ruby/object:Gem::Version
|
@@ -30,7 +27,6 @@ dependencies:
|
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: archive-tar-minitar
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
31
|
- - ~>
|
36
32
|
- !ruby/object:Gem::Version
|
@@ -38,7 +34,6 @@ dependencies:
|
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
38
|
- - ~>
|
44
39
|
- !ruby/object:Gem::Version
|
@@ -48,35 +43,36 @@ description: google_plus_archiver is a simple command-line tool to archive Googl
|
|
48
43
|
email: mort.yao@gmail.com
|
49
44
|
executables:
|
50
45
|
- gplus-get
|
46
|
+
- gplus-image-get
|
51
47
|
extensions: []
|
52
48
|
extra_rdoc_files: []
|
53
49
|
files:
|
54
50
|
- bin/gplus-get
|
55
51
|
- lib/google_plus_archiver.rb
|
56
52
|
- lib/google_plus_archiver/version.rb
|
53
|
+
- bin/gplus-image-get
|
57
54
|
homepage: https://github.com/soimort/google_plus_archiver
|
58
55
|
licenses:
|
59
56
|
- MIT
|
57
|
+
metadata: {}
|
60
58
|
post_install_message:
|
61
59
|
rdoc_options: []
|
62
60
|
require_paths:
|
63
61
|
- lib
|
64
62
|
required_ruby_version: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
63
|
requirements:
|
67
|
-
- -
|
64
|
+
- - '>='
|
68
65
|
- !ruby/object:Gem::Version
|
69
66
|
version: '0'
|
70
67
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
71
|
-
none: false
|
72
68
|
requirements:
|
73
|
-
- -
|
69
|
+
- - '>='
|
74
70
|
- !ruby/object:Gem::Version
|
75
71
|
version: '0'
|
76
72
|
requirements: []
|
77
73
|
rubyforge_project:
|
78
|
-
rubygems_version:
|
74
|
+
rubygems_version: 2.0.2
|
79
75
|
signing_key:
|
80
|
-
specification_version:
|
76
|
+
specification_version: 4
|
81
77
|
summary: A simple command-line tool to archive Google+ profiles.
|
82
78
|
test_files: []
|