squarepusher 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +29 -0
- data/bin/squarepusher +114 -87
- data/lib/squarepusher/cli.rb +167 -0
- data/lib/squarepusher/core.rb +214 -0
- data/lib/squarepusher/version.rb +1 -1
- data/lib/squarepusher.rb +209 -142
- metadata +8 -9
data/README.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
squarepusher downloads Flickr photosets to your local hard disk.
|
2
|
+
|
3
|
+
if you are using rubygems, get started with:
|
4
|
+
`gem install squarepusher`
|
5
|
+
|
6
|
+
you can easily start by creating a file ~/.squarepusher.yaml with this content:
|
7
|
+
|
8
|
+
key: consumer key
|
9
|
+
secret: consumer secret
|
10
|
+
token: access token
|
11
|
+
token_secret: access token secret
|
12
|
+
|
13
|
+
then run `squarepusher` with the action that you're interested in, foe example:
|
14
|
+
|
15
|
+
`squarepusher list_sets`
|
16
|
+
|
17
|
+
lists all of your flickr photosets.
|
18
|
+
|
19
|
+
there are a few different ways to generate an access token. assuming you have a consumer key, here is one way:
|
20
|
+
[http://www.flickr.com/services/api/auth.howto.mobile.html](http://www.flickr.com/services/api/auth.howto.mobile.html "flickr mobile auth instructions")
|
21
|
+
|
22
|
+
dependencies:
|
23
|
+
|
24
|
+
* [flickraw](https://github.com/hanklords/flickraw "flickraw")
|
25
|
+
|
26
|
+
|
27
|
+
behaviors of note:
|
28
|
+
|
29
|
+
* photoset and image names are normalized using this: `name.gsub(/[^\w_+\.=-]+/, '-')`
|
data/bin/squarepusher
CHANGED
@@ -3,91 +3,118 @@
|
|
3
3
|
require 'rubygems'
|
4
4
|
require 'squarepusher'
|
5
5
|
|
6
|
-
|
7
|
-
$stderr.puts "ERROR - #{msg}"
|
8
|
-
end
|
6
|
+
Squarepusher::CLI.main
|
9
7
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
8
|
+
# require 'rubygems'
|
9
|
+
# require 'squarepusher'
|
10
|
+
#
|
11
|
+
# def error_msg(msg)
|
12
|
+
# $stderr.puts "ERROR - #{msg}"
|
13
|
+
# end
|
14
|
+
#
|
15
|
+
# def fail(msg, code=1)
|
16
|
+
# error_msg(msg)
|
17
|
+
# exit code
|
18
|
+
# end
|
19
|
+
#
|
20
|
+
#
|
21
|
+
# actions = {}
|
22
|
+
#
|
23
|
+
# actions[:list_sets] = lambda do |client, args|
|
24
|
+
# client.each_photoset do |pset|
|
25
|
+
# puts Squarepusher.describe_photoset(pset)
|
26
|
+
# end
|
27
|
+
# end
|
28
|
+
#
|
29
|
+
# actions[:grab_all_sets] = lambda do |client, args|
|
30
|
+
# if args.size != 1
|
31
|
+
# fail("expected: <output_dir>")
|
32
|
+
# end
|
33
|
+
#
|
34
|
+
# output_dir = args[0]
|
35
|
+
#
|
36
|
+
# total_results = {}
|
37
|
+
# client.each_photoset do |pset|
|
38
|
+
# pset_description = Squarepusher.describe_photoset(pset)
|
39
|
+
# puts "[set] #{pset_description}"
|
40
|
+
# pset_results = client.download_photoset(pset, output_dir)
|
41
|
+
# puts "[results] #{pset_description} #{pset_results.inspect}"
|
42
|
+
# total_results[pset_description] = pset_results
|
43
|
+
# puts
|
44
|
+
# end
|
45
|
+
#
|
46
|
+
# total_results.each_pair do |k, v|
|
47
|
+
# puts "#{k}: #{v.inspect}"
|
48
|
+
# end
|
49
|
+
# end
|
50
|
+
#
|
51
|
+
# actions[:grab_set] = lambda do |client, args|
|
52
|
+
# if args.size != 2
|
53
|
+
# fail("expected: <photoset_id> <output_dir>")
|
54
|
+
# end
|
55
|
+
#
|
56
|
+
# pset_id, output_dir = args
|
57
|
+
# pset = client.get_photoset(pset_id)
|
58
|
+
# results = client.download_photoset(pset, output_dir)
|
59
|
+
# puts results.inspect
|
60
|
+
# end
|
61
|
+
#
|
62
|
+
# actions[:find_set] = lambda do |client, args|
|
63
|
+
# if args.size != 1
|
64
|
+
# fail("expected: <photoset_id>")
|
65
|
+
# end
|
66
|
+
#
|
67
|
+
# pset_id, output_dir = args
|
68
|
+
# pset = client.get_photoset(pset_id)
|
69
|
+
# puts Squarepusher.describe_photoset(pset)
|
70
|
+
# end
|
71
|
+
#
|
72
|
+
#
|
73
|
+
# def action_str(actions)
|
74
|
+
# actions.keys.join(",")
|
75
|
+
# end
|
76
|
+
#
|
77
|
+
# def size_str
|
78
|
+
# Squarepusher.sizes.join(",")
|
79
|
+
# end
|
80
|
+
#
|
81
|
+
#
|
82
|
+
# options = { :size => :original }
|
83
|
+
# args = nil
|
84
|
+
# action = nil
|
85
|
+
# client = nil
|
86
|
+
# OptionParser.new do |opts|
|
87
|
+
# opts.banner = "USAGE: squarepusher [options] <api-key> <api-secret> <token> <action:(#{action_str(actions)})> [args]"
|
88
|
+
#
|
89
|
+
# opts.on("-o", "--overwrite", "overwrite already downloaded files") do |v|
|
90
|
+
# options[:overwrite] = v
|
91
|
+
# end
|
92
|
+
#
|
93
|
+
# opts.on("-s", "--size SIZE", "size of photos to download (#{size_str()})") do |v|
|
94
|
+
# options[:size] = v.to_sym
|
95
|
+
# end
|
96
|
+
#
|
97
|
+
# opts.on("-v", "--verbose", "log verbosely") do |v|
|
98
|
+
# options[:verbose] = v
|
99
|
+
# end
|
100
|
+
#
|
101
|
+
# opts.parse!(ARGV)
|
102
|
+
#
|
103
|
+
# if ARGV.size < 4
|
104
|
+
# $stderr.puts opts
|
105
|
+
# exit 1
|
106
|
+
# end
|
107
|
+
#
|
108
|
+
# key, secret, token, action = ARGV.slice!(0...4)
|
109
|
+
# args = ARGV
|
110
|
+
#
|
111
|
+
# action = action.to_sym
|
112
|
+
# if not actions.has_key?(action)
|
113
|
+
# $stderr.puts opts
|
114
|
+
# exit 2
|
115
|
+
# end
|
116
|
+
#
|
117
|
+
# client = Squarepusher::Client.new(key, secret, token, options)
|
118
|
+
# end
|
119
|
+
#
|
120
|
+
# actions[action].call(client, args || [])
|
@@ -0,0 +1,167 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
|
6
|
+
module Squarepusher
|
7
|
+
|
8
|
+
class CLI
|
9
|
+
|
10
|
+
class << self
|
11
|
+
|
12
|
+
def error_msg(msg)
|
13
|
+
$stderr.puts "ERROR - #{msg}"
|
14
|
+
end
|
15
|
+
|
16
|
+
def fail(msg, code=1)
|
17
|
+
error_msg(msg)
|
18
|
+
exit code
|
19
|
+
end
|
20
|
+
|
21
|
+
def parse_config(path)
|
22
|
+
parsed = nil
|
23
|
+
open(path) do |stream|
|
24
|
+
parsed = YAML.load(stream)
|
25
|
+
end
|
26
|
+
if not parsed.respond_to?(:has_key?)
|
27
|
+
raise Exception.new("invalid YAML in #{path}; parsed #{parsed.inspect}")
|
28
|
+
end
|
29
|
+
puts "parsed config: #{parsed.inspect}"
|
30
|
+
required_keys = ['key', 'secret', 'token', 'token_secret']
|
31
|
+
missing_keys = []
|
32
|
+
vals = []
|
33
|
+
required_keys.each do |k|
|
34
|
+
v = parsed.fetch(k, nil)
|
35
|
+
if not v.nil?
|
36
|
+
vals << v
|
37
|
+
else
|
38
|
+
missing_keys << k
|
39
|
+
end
|
40
|
+
end
|
41
|
+
if not missing_keys.empty?
|
42
|
+
raise Exception.new("missing keys in #{path}: #{missing_keys.inspect}")
|
43
|
+
end
|
44
|
+
vals
|
45
|
+
end
|
46
|
+
|
47
|
+
def main
|
48
|
+
actions = {}
|
49
|
+
|
50
|
+
actions[:list_sets] = lambda do |client, args|
|
51
|
+
client.each_photoset do |pset|
|
52
|
+
puts Squarepusher.describe_photoset(pset)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
actions[:grab_all_sets] = lambda do |client, args|
|
57
|
+
if args.size != 1
|
58
|
+
fail("expected: <output_dir>")
|
59
|
+
end
|
60
|
+
|
61
|
+
output_dir = args[0]
|
62
|
+
|
63
|
+
total_results = {}
|
64
|
+
client.each_photoset do |pset|
|
65
|
+
pset_description = Squarepusher.describe_photoset(pset)
|
66
|
+
puts "[set] #{pset_description}"
|
67
|
+
pset_results = client.download_photoset(pset, output_dir)
|
68
|
+
puts "[results] #{pset_description} #{pset_results.inspect}"
|
69
|
+
total_results[pset_description] = pset_results
|
70
|
+
puts
|
71
|
+
end
|
72
|
+
|
73
|
+
total_results.each_pair do |k, v|
|
74
|
+
puts "#{k}: #{v.inspect}"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
actions[:grab_set] = lambda do |client, args|
|
79
|
+
if args.size != 2
|
80
|
+
fail("expected: <photoset_id> <output_dir>")
|
81
|
+
end
|
82
|
+
|
83
|
+
pset_id, output_dir = args
|
84
|
+
pset = client.get_photoset(pset_id)
|
85
|
+
results = client.download_photoset(pset, output_dir)
|
86
|
+
puts results.inspect
|
87
|
+
end
|
88
|
+
|
89
|
+
actions[:find_set] = lambda do |client, args|
|
90
|
+
if args.size != 1
|
91
|
+
fail("expected: <photoset_id>")
|
92
|
+
end
|
93
|
+
|
94
|
+
pset_id, output_dir = args
|
95
|
+
pset = client.get_photoset(pset_id)
|
96
|
+
puts Squarepusher.describe_photoset(pset)
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
def action_str(actions)
|
101
|
+
actions.keys.join(",")
|
102
|
+
end
|
103
|
+
|
104
|
+
def size_str
|
105
|
+
Squarepusher.sizes.join(",")
|
106
|
+
end
|
107
|
+
|
108
|
+
options = { :size => :original }
|
109
|
+
args = nil
|
110
|
+
action = nil
|
111
|
+
client = nil
|
112
|
+
config_path = "#{ENV['HOME']}/.squarepusher.yaml"
|
113
|
+
OptionParser.new do |opts|
|
114
|
+
opts.banner = "USAGE: squarepusher [options] <action:(#{action_str(actions)})> [args]"
|
115
|
+
|
116
|
+
opts.on("-c", "--config", "path to config file (defaults to #{config_path})") do |v|
|
117
|
+
options[:config] = v
|
118
|
+
end
|
119
|
+
|
120
|
+
opts.on("-o", "--overwrite", "overwrite already downloaded files") do |v|
|
121
|
+
options[:overwrite] = v
|
122
|
+
end
|
123
|
+
|
124
|
+
opts.on("-s", "--size SIZE", "size of photos to download (#{size_str()})") do |v|
|
125
|
+
options[:size] = v.to_sym
|
126
|
+
end
|
127
|
+
|
128
|
+
opts.on("-v", "--verbose", "log verbosely") do |v|
|
129
|
+
options[:verbose] = v
|
130
|
+
end
|
131
|
+
|
132
|
+
opts.parse!(ARGV)
|
133
|
+
|
134
|
+
if ARGV.size < 1
|
135
|
+
$stderr.puts opts
|
136
|
+
exit 1
|
137
|
+
end
|
138
|
+
|
139
|
+
action = ARGV.delete_at(0)
|
140
|
+
args = ARGV
|
141
|
+
|
142
|
+
action = action.to_sym
|
143
|
+
if not actions.has_key?(action)
|
144
|
+
$stderr.puts opts
|
145
|
+
exit 2
|
146
|
+
end
|
147
|
+
|
148
|
+
config_path = options[:config] || config_path
|
149
|
+
if not File.exists?(config_path)
|
150
|
+
fail "#{config_path} not found"
|
151
|
+
else
|
152
|
+
puts "parsing #{config_path}"
|
153
|
+
end
|
154
|
+
|
155
|
+
key, secret, token, token_secret = parse_config(config_path)
|
156
|
+
client = Squarepusher::Client.new(key, secret, token, token_secret, options)
|
157
|
+
end
|
158
|
+
|
159
|
+
puts "args: #{args.inspect}"
|
160
|
+
actions[action].call(client, args || [])
|
161
|
+
end
|
162
|
+
|
163
|
+
end
|
164
|
+
|
165
|
+
end
|
166
|
+
|
167
|
+
end
|
@@ -0,0 +1,214 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'optparse'
|
3
|
+
|
4
|
+
FlickRawOptions = {}
|
5
|
+
|
6
|
+
$fileutils = FileUtils::Verbose
|
7
|
+
|
8
|
+
# TODO: detect "photo unavailable" and don't download image
|
9
|
+
UNAVAILABLE_URL = 'http://l.yimg.com/g/images/photo_unavailable.gif'
|
10
|
+
|
11
|
+
# TODO: keep track of failures
|
12
|
+
|
13
|
+
module Squarepusher
|
14
|
+
|
15
|
+
class << self
|
16
|
+
|
17
|
+
def describe_photoset(pset)
|
18
|
+
"#{pset.id} '#{pset.title}'"
|
19
|
+
end
|
20
|
+
|
21
|
+
def normalize(name)
|
22
|
+
name.gsub(/[^a-zA-Z0-9_+\.=-]+/, '-')
|
23
|
+
end
|
24
|
+
|
25
|
+
def sizes
|
26
|
+
return [:original, :large, :medium_640, :medium_500, :small, :thumb, :small_square]
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
class Client
|
32
|
+
|
33
|
+
def initialize(key, secret, token, token_secret, args={})
|
34
|
+
@overwrite = args[:overwrite] || false
|
35
|
+
@verbose = args[:verbose] || false
|
36
|
+
|
37
|
+
FlickRawOptions['timeout'] = args[:timeout] || 5
|
38
|
+
|
39
|
+
require 'flickraw'
|
40
|
+
|
41
|
+
FlickRaw.api_key = key
|
42
|
+
FlickRaw.shared_secret = secret
|
43
|
+
|
44
|
+
# flickr.auth.checkToken(:auth_token => token)
|
45
|
+
flickr.access_token = token
|
46
|
+
flickr.access_secret = token_secret
|
47
|
+
|
48
|
+
size = args[:size] || :small_square
|
49
|
+
|
50
|
+
@url_for_photo = case size
|
51
|
+
when :original
|
52
|
+
lambda { |p| FlickRaw.url_o(p) }
|
53
|
+
when :large
|
54
|
+
lambda { |p| FlickRaw.url_b(p) }
|
55
|
+
when :medium_640
|
56
|
+
lambda { |p| FlickRaw.url_z(p) }
|
57
|
+
when :medium_500
|
58
|
+
lambda { |p| FlickRaw.url(p) }
|
59
|
+
when :small
|
60
|
+
lambda { |p| FlickRaw.url_m(p) }
|
61
|
+
when :thumb
|
62
|
+
lambda { |p| FlickRaw.url_t(p) }
|
63
|
+
when :small_square
|
64
|
+
lambda { |p| FlickRaw.url_s(p) }
|
65
|
+
else
|
66
|
+
raise Exception("unrecognized size: #{size}")
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def get_photoset(pset_id)
|
71
|
+
flickr.photosets.getInfo(:photoset_id => pset_id)
|
72
|
+
end
|
73
|
+
|
74
|
+
def each_photoset
|
75
|
+
flickr.photosets.getList.each do |pset|
|
76
|
+
yield pset
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def download_photoset(photoset, output_dir)
|
81
|
+
dirname = Squarepusher.normalize(photoset.title)
|
82
|
+
set_dir = File.join(output_dir, dirname)
|
83
|
+
$fileutils.mkdir_p set_dir
|
84
|
+
|
85
|
+
# handles socket timeout when loading photoset info
|
86
|
+
results = {}
|
87
|
+
status, photoset_result = handle_error { flickr.photosets.getPhotos(:photoset_id => photoset.id, :extras => "original_format,url_o")["photo"] }
|
88
|
+
if status == :error
|
89
|
+
error photoset_result
|
90
|
+
results[status] = 1
|
91
|
+
return results
|
92
|
+
else
|
93
|
+
photos = photoset_result
|
94
|
+
end
|
95
|
+
|
96
|
+
photos.each do |p|
|
97
|
+
# TODO: use name of file in url since titles can be duplicate
|
98
|
+
# copies string?
|
99
|
+
name = "#{p.id}"
|
100
|
+
if not p.title.empty?
|
101
|
+
normalized_title = Squarepusher.normalize(p.title)
|
102
|
+
name << "-#{normalized_title}"
|
103
|
+
end
|
104
|
+
|
105
|
+
url = @url_for_photo[p]
|
106
|
+
|
107
|
+
# TODO: only add .jpg suffix if it's not in the image name already
|
108
|
+
path = File.join(set_dir, name)
|
109
|
+
path << ".jpg" if not path =~ /.jpg$/
|
110
|
+
|
111
|
+
if not @overwrite and File.exists?(path)
|
112
|
+
debug "#{path} exists; skipping"
|
113
|
+
result = :exists
|
114
|
+
else
|
115
|
+
result = download_image(url, path)
|
116
|
+
end
|
117
|
+
|
118
|
+
if results.has_key?(result)
|
119
|
+
results[result] = results[result] + 1
|
120
|
+
else
|
121
|
+
results[result] = 1
|
122
|
+
end
|
123
|
+
end
|
124
|
+
results
|
125
|
+
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
# was originally sending this to stderr, but the difference in flushing between
|
130
|
+
# stdout and stderr was creating logging confusion
|
131
|
+
def error(msg)
|
132
|
+
puts "ERROR: #{msg}"
|
133
|
+
end
|
134
|
+
|
135
|
+
def debug(msg)
|
136
|
+
puts msg if @verbose
|
137
|
+
end
|
138
|
+
|
139
|
+
def info(msg)
|
140
|
+
puts msg
|
141
|
+
end
|
142
|
+
|
143
|
+
def download_image(url, path, args={})
|
144
|
+
if url == UNAVAILABLE_URL
|
145
|
+
redirects = args[:redirects]
|
146
|
+
msg = "#{url} unavailable"
|
147
|
+
msg << "; redirects: #{redirects}" if not redirects.nil?
|
148
|
+
debug msg
|
149
|
+
return :unavailable
|
150
|
+
end
|
151
|
+
|
152
|
+
result = :unknown
|
153
|
+
uri = URI.parse(url)
|
154
|
+
begin
|
155
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
156
|
+
full_path = uri.request_uri
|
157
|
+
response = http.get(full_path)
|
158
|
+
|
159
|
+
# puts response.inspect
|
160
|
+
case response
|
161
|
+
when Net::HTTPError
|
162
|
+
result = :error
|
163
|
+
when Net::HTTPFound
|
164
|
+
redirects = args[:redirects] || []
|
165
|
+
redirects << url
|
166
|
+
location = response["location"]
|
167
|
+
if not location =~ /^http.*/
|
168
|
+
location = "#{uri.scheme}://#{uri.host}:#{uri.port}#{location}"
|
169
|
+
end
|
170
|
+
result = download_image(location, path, :redirects => redirects)
|
171
|
+
else
|
172
|
+
debug "#{url} -> #{path}"
|
173
|
+
if @verbose
|
174
|
+
response.each_header do |k, v|
|
175
|
+
puts "#{k}: #{v}"
|
176
|
+
end
|
177
|
+
puts
|
178
|
+
end
|
179
|
+
|
180
|
+
# detects weird 404 or 504 responses included in body but not HTTP status
|
181
|
+
content_type = response["content-type"]
|
182
|
+
if not (content_type =~ /text\/\w+/).nil?
|
183
|
+
error "unexpected content type: #{content_type}"
|
184
|
+
result = :error
|
185
|
+
else
|
186
|
+
open(path, 'w') do |out|
|
187
|
+
out << response.body
|
188
|
+
end
|
189
|
+
result = :success
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
rescue Exception => e
|
194
|
+
error e
|
195
|
+
result = :error
|
196
|
+
end
|
197
|
+
|
198
|
+
return result
|
199
|
+
end
|
200
|
+
|
201
|
+
def handle_error
|
202
|
+
raise Exception("block required") if not block_given?
|
203
|
+
begin
|
204
|
+
block_result = yield
|
205
|
+
result = [:success, block_result]
|
206
|
+
rescue
|
207
|
+
result = [:error, $!]
|
208
|
+
end
|
209
|
+
return result
|
210
|
+
end
|
211
|
+
|
212
|
+
end
|
213
|
+
|
214
|
+
end
|
data/lib/squarepusher/version.rb
CHANGED
data/lib/squarepusher.rb
CHANGED
@@ -1,143 +1,210 @@
|
|
1
|
-
require '
|
2
|
-
require '
|
1
|
+
require 'squarepusher/cli'
|
2
|
+
require 'squarepusher/core'
|
3
|
+
require 'squarepusher/version'
|
3
4
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
#
|
9
|
-
|
10
|
-
|
11
|
-
# TODO:
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
5
|
+
# require 'fileutils'
|
6
|
+
# require 'optparse'
|
7
|
+
#
|
8
|
+
# FlickRawOptions = {}
|
9
|
+
#
|
10
|
+
# $fileutils = FileUtils::Verbose
|
11
|
+
#
|
12
|
+
# # TODO: detect "photo unavailable" and don't download image
|
13
|
+
# UNAVAILABLE_URL = 'http://l.yimg.com/g/images/photo_unavailable.gif'
|
14
|
+
#
|
15
|
+
# # TODO: keep track of failures
|
16
|
+
#
|
17
|
+
# module Squarepusher
|
18
|
+
#
|
19
|
+
# class << self
|
20
|
+
#
|
21
|
+
# def describe_photoset(pset)
|
22
|
+
# "#{pset.id} '#{pset.title}'"
|
23
|
+
# end
|
24
|
+
#
|
25
|
+
# def normalize(name)
|
26
|
+
# name.gsub(/[^a-zA-Z0-9_+\.=-]+/, '-')
|
27
|
+
# end
|
28
|
+
#
|
29
|
+
# def sizes
|
30
|
+
# return [:original, :large, :medium_640, :medium_500, :small, :thumb, :small_square]
|
31
|
+
# end
|
32
|
+
#
|
33
|
+
# end
|
34
|
+
#
|
35
|
+
# class Client
|
36
|
+
#
|
37
|
+
# def initialize(key, secret, token, args={})
|
38
|
+
# @overwrite = args[:overwrite] || false
|
39
|
+
# @verbose = args[:verbose] || false
|
40
|
+
#
|
41
|
+
# FlickRawOptions['timeout'] = args[:timeout] || 5
|
42
|
+
#
|
43
|
+
# require 'flickraw'
|
44
|
+
#
|
45
|
+
# FlickRaw.api_key = key
|
46
|
+
# FlickRaw.shared_secret = secret
|
47
|
+
#
|
48
|
+
# flickr.auth.checkToken(:auth_token => token)
|
49
|
+
#
|
50
|
+
# size = args[:size] || :small_square
|
51
|
+
#
|
52
|
+
# @url_for_photo = case size
|
53
|
+
# when :original
|
54
|
+
# lambda { |p| FlickRaw.url_o(p) }
|
55
|
+
# when :large
|
56
|
+
# lambda { |p| FlickRaw.url_b(p) }
|
57
|
+
# when :medium_640
|
58
|
+
# lambda { |p| FlickRaw.url_z(p) }
|
59
|
+
# when :medium_500
|
60
|
+
# lambda { |p| FlickRaw.url(p) }
|
61
|
+
# when :small
|
62
|
+
# lambda { |p| FlickRaw.url_m(p) }
|
63
|
+
# when :thumb
|
64
|
+
# lambda { |p| FlickRaw.url_t(p) }
|
65
|
+
# when :small_square
|
66
|
+
# lambda { |p| FlickRaw.url_s(p) }
|
67
|
+
# else
|
68
|
+
# raise Exception("unrecognized size: #{size}")
|
69
|
+
# end
|
70
|
+
# end
|
71
|
+
#
|
72
|
+
# def get_photoset(pset_id)
|
73
|
+
# flickr.photosets.getInfo(:photoset_id => pset_id)
|
74
|
+
# end
|
75
|
+
#
|
76
|
+
# def each_photoset
|
77
|
+
# flickr.photosets.getList.each do |pset|
|
78
|
+
# yield pset
|
79
|
+
# end
|
80
|
+
# end
|
81
|
+
#
|
82
|
+
# def download_photoset(photoset, output_dir)
|
83
|
+
# dirname = Squarepusher.normalize(photoset.title)
|
84
|
+
# set_dir = File.join(output_dir, dirname)
|
85
|
+
# $fileutils.mkdir_p set_dir
|
86
|
+
#
|
87
|
+
# # handles socket timeout when loading photoset info
|
88
|
+
# results = {}
|
89
|
+
# status, photoset_result = handle_error { flickr.photosets.getPhotos(:photoset_id => photoset.id, :extras => "original_format,url_o")["photo"] }
|
90
|
+
# if status == :error
|
91
|
+
# error photoset_result
|
92
|
+
# results[status] = 1
|
93
|
+
# return results
|
94
|
+
# else
|
95
|
+
# photos = photoset_result
|
96
|
+
# end
|
97
|
+
#
|
98
|
+
# photos.each do |p|
|
99
|
+
# # puts p.inspect
|
100
|
+
#
|
101
|
+
# # TODO: use name of file in url since titles can be duplicate
|
102
|
+
# # copies string?
|
103
|
+
# name = "#{p.id}"
|
104
|
+
# if not p.title.empty?
|
105
|
+
# normalized_title = Squarepusher.normalize(p.title)
|
106
|
+
# name << "-#{normalized_title}"
|
107
|
+
# end
|
108
|
+
#
|
109
|
+
# url = @url_for_photo[p]
|
110
|
+
#
|
111
|
+
# # TODO: only add .jpg suffix if it's not in the image name already
|
112
|
+
# path = File.join(set_dir, name)
|
113
|
+
# path << ".jpg" if not path =~ /.jpg$/
|
114
|
+
#
|
115
|
+
# if not @overwrite and File.exists?(path)
|
116
|
+
# puts "#{path} exists; skipping"
|
117
|
+
# result = :exists
|
118
|
+
# else
|
119
|
+
# result = download_image(url, path)
|
120
|
+
# end
|
121
|
+
#
|
122
|
+
# if results.has_key?(result)
|
123
|
+
# results[result] = results[result] + 1
|
124
|
+
# else
|
125
|
+
# results[result] = 1
|
126
|
+
# end
|
127
|
+
# end
|
128
|
+
# results
|
129
|
+
# end
|
130
|
+
#
|
131
|
+
# private
|
132
|
+
#
|
133
|
+
# # was originally sending this to stderr, but the difference in flushing between
|
134
|
+
# # stdout and stderr was creating logging confusion
|
135
|
+
# def error(msg)
|
136
|
+
# puts "ERROR: #{msg}"
|
137
|
+
# end
|
138
|
+
#
|
139
|
+
# def download_image(url, path, args={})
|
140
|
+
# if url == UNAVAILABLE_URL
|
141
|
+
# redirects = args[:redirects]
|
142
|
+
# msg = "#{url} unavailable"
|
143
|
+
# msg << "; redirects: #{redirects}" if not redirects.nil?
|
144
|
+
# puts msg
|
145
|
+
# return :unavailable
|
146
|
+
# end
|
147
|
+
#
|
148
|
+
# result = :unknown
|
149
|
+
# uri = URI.parse(url)
|
150
|
+
# begin
|
151
|
+
# Net::HTTP.start(uri.host, uri.port) do |http|
|
152
|
+
# full_path = uri.request_uri
|
153
|
+
# response = http.get(full_path)
|
154
|
+
#
|
155
|
+
# # puts response.inspect
|
156
|
+
# case response
|
157
|
+
# when Net::HTTPError
|
158
|
+
# result = :error
|
159
|
+
# when Net::HTTPFound
|
160
|
+
# redirects = args[:redirects] || []
|
161
|
+
# redirects << url
|
162
|
+
# location = response["location"]
|
163
|
+
# if not location =~ /^http.*/
|
164
|
+
# location = "#{uri.scheme}://#{uri.host}:#{uri.port}#{location}"
|
165
|
+
# end
|
166
|
+
# result = download_image(location, path, :redirects => redirects)
|
167
|
+
# else
|
168
|
+
# puts "#{url} -> #{path}"
|
169
|
+
# if @verbose
|
170
|
+
# response.each_header do |k, v|
|
171
|
+
# puts "#{k}: #{v}"
|
172
|
+
# end
|
173
|
+
# puts
|
174
|
+
# end
|
175
|
+
#
|
176
|
+
# # detects weird 404 or 504 responses included in body but not HTTP status
|
177
|
+
# content_type = response["content-type"]
|
178
|
+
# if not (content_type =~ /text\/\w+/).nil?
|
179
|
+
# error "unexpected content type: #{content_type}"
|
180
|
+
# result = :error
|
181
|
+
# else
|
182
|
+
# open(path, 'w') do |out|
|
183
|
+
# out << response.body
|
184
|
+
# end
|
185
|
+
# result = :success
|
186
|
+
# end
|
187
|
+
# end
|
188
|
+
# end
|
189
|
+
# rescue Exception => e
|
190
|
+
# error e
|
191
|
+
# result = :error
|
192
|
+
# end
|
193
|
+
#
|
194
|
+
# return result
|
195
|
+
# end
|
196
|
+
#
|
197
|
+
# def handle_error
|
198
|
+
# raise Exception("block required") if not block_given?
|
199
|
+
# begin
|
200
|
+
# block_result = yield
|
201
|
+
# result = [:success, block_result]
|
202
|
+
# rescue
|
203
|
+
# result = [:error, $!]
|
204
|
+
# end
|
205
|
+
# return result
|
206
|
+
# end
|
207
|
+
#
|
208
|
+
# end
|
209
|
+
#
|
210
|
+
# end
|
metadata
CHANGED
@@ -1,13 +1,12 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: squarepusher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
prerelease:
|
4
|
+
prerelease: false
|
6
5
|
segments:
|
7
6
|
- 0
|
8
7
|
- 0
|
9
|
-
-
|
10
|
-
version: 0.0.
|
8
|
+
- 4
|
9
|
+
version: 0.0.4
|
11
10
|
platform: ruby
|
12
11
|
authors:
|
13
12
|
- mhawthorne
|
@@ -15,7 +14,7 @@ autorequire:
|
|
15
14
|
bindir: bin
|
16
15
|
cert_chain: []
|
17
16
|
|
18
|
-
date: 2011-
|
17
|
+
date: 2011-10-24 00:00:00 -07:00
|
19
18
|
default_executable:
|
20
19
|
dependencies:
|
21
20
|
- !ruby/object:Gem::Dependency
|
@@ -26,7 +25,6 @@ dependencies:
|
|
26
25
|
requirements:
|
27
26
|
- - ">="
|
28
27
|
- !ruby/object:Gem::Version
|
29
|
-
hash: 3
|
30
28
|
segments:
|
31
29
|
- 0
|
32
30
|
version: "0"
|
@@ -43,9 +41,12 @@ extra_rdoc_files: []
|
|
43
41
|
files:
|
44
42
|
- .gitignore
|
45
43
|
- Gemfile
|
44
|
+
- README.md
|
46
45
|
- Rakefile
|
47
46
|
- bin/squarepusher
|
48
47
|
- lib/squarepusher.rb
|
48
|
+
- lib/squarepusher/cli.rb
|
49
|
+
- lib/squarepusher/core.rb
|
49
50
|
- lib/squarepusher/version.rb
|
50
51
|
- squarepusher.gemspec
|
51
52
|
has_rdoc: true
|
@@ -62,7 +63,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
62
63
|
requirements:
|
63
64
|
- - ">="
|
64
65
|
- !ruby/object:Gem::Version
|
65
|
-
hash: 3
|
66
66
|
segments:
|
67
67
|
- 0
|
68
68
|
version: "0"
|
@@ -71,14 +71,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
71
71
|
requirements:
|
72
72
|
- - ">="
|
73
73
|
- !ruby/object:Gem::Version
|
74
|
-
hash: 3
|
75
74
|
segments:
|
76
75
|
- 0
|
77
76
|
version: "0"
|
78
77
|
requirements: []
|
79
78
|
|
80
79
|
rubyforge_project: nowarning
|
81
|
-
rubygems_version: 1.
|
80
|
+
rubygems_version: 1.3.7
|
82
81
|
signing_key:
|
83
82
|
specification_version: 3
|
84
83
|
summary: downloads photos from flickr
|