s3ranger 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +4 -2
- data/bin/s3ranger +3 -69
- data/lib/s3ranger/cli.rb +344 -0
- data/lib/s3ranger/sync.rb +16 -16
- data/lib/s3ranger/util.rb +9 -0
- data/lib/s3ranger/version.rb +1 -1
- data/s3ranger.gemspec +1 -0
- data/spec/main_spec.rb +1 -2
- metadata +21 -6
- data/lib/s3ranger/cmd.rb +0 -112
- data/lib/s3ranger/commands.rb +0 -114
data/Gemfile.lock
CHANGED
@@ -1,16 +1,18 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
s3ranger (0.
|
4
|
+
s3ranger (0.2.0)
|
5
5
|
aws-sdk
|
6
|
+
cmdparse
|
6
7
|
|
7
8
|
GEM
|
8
9
|
remote: https://rubygems.org/
|
9
10
|
specs:
|
10
|
-
aws-sdk (1.
|
11
|
+
aws-sdk (1.16.0)
|
11
12
|
json (~> 1.4)
|
12
13
|
nokogiri (< 1.6.0)
|
13
14
|
uuidtools (~> 2.1)
|
15
|
+
cmdparse (2.0.5)
|
14
16
|
columnize (0.3.6)
|
15
17
|
debugger (1.6.1)
|
16
18
|
columnize (>= 0.3.1)
|
data/bin/s3ranger
CHANGED
@@ -4,7 +4,7 @@ $:.unshift(File.dirname(__FILE__) + '/../lib') unless $:.include?(File.dirname(_
|
|
4
4
|
|
5
5
|
require "s3ranger/exceptions"
|
6
6
|
require "s3ranger/config"
|
7
|
-
require "s3ranger/
|
7
|
+
require "s3ranger/cli"
|
8
8
|
|
9
9
|
conf = S3Ranger::Config.new
|
10
10
|
|
@@ -24,82 +24,16 @@ rescue S3Ranger::NoConfigFound => exc
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
-
|
28
27
|
# Step aside, the star of this show is here. Let's try to create the
|
29
28
|
# environment to run the requested command. And feed the user back if
|
30
29
|
# information needed was not enough
|
31
30
|
begin
|
32
|
-
S3Ranger::
|
33
|
-
|
31
|
+
S3Ranger::CLI::run conf
|
34
32
|
rescue S3Ranger::FailureFeedback => exc
|
35
33
|
$stderr.puts exc.message
|
36
34
|
exit 1
|
37
|
-
|
38
35
|
rescue S3Ranger::WrongUsage => exc
|
39
36
|
name = $0.split('/').last
|
40
|
-
|
41
|
-
$stderr.puts <<"ENDUSAGE"
|
42
|
-
Usage: #{name} [options] <command> [arg(s)]
|
43
|
-
|
44
|
-
Global Options:
|
45
|
-
-h, --help
|
46
|
-
-v, --verbose
|
47
|
-
-n, --dryrun
|
48
|
-
-d, --debug
|
49
|
-
--progress
|
50
|
-
|
51
|
-
#{name} listbuckets
|
52
|
-
List all available buckets
|
53
|
-
|
54
|
-
#{name} createbucket <bucket>
|
55
|
-
Creates a new bucket
|
56
|
-
|
57
|
-
Options:
|
58
|
-
|
59
|
-
-a <ACL>, --acl=(private|public_read|public_read_write)
|
60
|
-
|
61
|
-
#{name} deletebucket [options] <bucket>
|
62
|
-
Removes an existing bucket
|
63
|
-
|
64
|
-
Options:
|
65
|
-
-f, --force Deletes non-empty buckets (BE CAREFUL)
|
66
|
-
|
67
|
-
#{name} list <bucket>[:prefix]
|
68
|
-
List content inside of bucket
|
69
|
-
|
70
|
-
if `prefix' is present, only content under `prefix' will be listed.
|
71
|
-
|
72
|
-
#{name} delete <bucket>:<key>
|
73
|
-
Removes `key` from `bucket`
|
74
|
-
|
75
|
-
#{name} url [options] <bucket>:<key>
|
76
|
-
Generates a presigned URL for an operation on the object named `key' found on
|
77
|
-
`bucket'.
|
78
|
-
|
79
|
-
Options:
|
80
|
-
--no-ssl
|
81
|
-
--expires-in=(<# of seconds> | [#d|#h|#m|#s])
|
82
|
-
|
83
|
-
#{name} get <bucket>:<key> <file>
|
84
|
-
Retrieves the remote `key` object from `bucket` and saves to the local path
|
85
|
-
specified in `file`
|
86
|
-
|
87
|
-
#{name} put <bucket>[:<key>] <file>
|
88
|
-
Uploads the file `file` to the `bucket` under the path `key`
|
89
|
-
|
90
|
-
#{name} sync <source> <destination>
|
91
|
-
|
92
|
-
One of source or destination must be remote and the other must be local,
|
93
|
-
where local points to a folder in the file system and remote conform to the
|
94
|
-
format `<bucket>[:<key>]`.
|
95
|
-
|
96
|
-
Options:
|
97
|
-
-e <pattern>, --exclude=<pattern>
|
98
|
-
-k, --keep
|
99
|
-
-d, --dry-run
|
100
|
-
|
101
|
-
ENDUSAGE
|
102
|
-
|
103
|
-
$stderr.puts "\nCurrent error:\n #{exc.msg}\n" if exc.msg
|
37
|
+
$stderr.puts "Error:\n #{exc.msg}\n" if exc.msg
|
104
38
|
exit exc.error_code
|
105
39
|
end
|
data/lib/s3ranger/cli.rb
ADDED
@@ -0,0 +1,344 @@
|
|
1
|
+
require 's3ranger/exceptions'
|
2
|
+
require 's3ranger/sync'
|
3
|
+
require 'aws/s3'
|
4
|
+
require 'cmdparse'
|
5
|
+
|
6
|
+
|
7
|
+
module S3Ranger
|
8
|
+
module CLI
|
9
|
+
|
10
|
+
AVAILABLE_ACLS = [:public_read, :public_read_write, :private]
|
11
|
+
|
12
|
+
AVAILABLE_METHODS = ['read', 'get', 'put', 'write', 'delete']
|
13
|
+
|
14
|
+
class ListBuckets < CmdParse::Command
|
15
|
+
def initialize
|
16
|
+
super 'listbuckets', false, false, false
|
17
|
+
|
18
|
+
@short_desc = "List all available buckets for your user"
|
19
|
+
end
|
20
|
+
|
21
|
+
def run s3, bucket, key, file, args
|
22
|
+
s3.buckets.each do |bkt|
|
23
|
+
puts "#{bkt.name}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class CreateBucket < CmdParse::Command
|
29
|
+
attr_accessor :acl
|
30
|
+
|
31
|
+
def initialize
|
32
|
+
super 'createbucket', false, false
|
33
|
+
|
34
|
+
@short_desc = "Create a new bucket under your user account"
|
35
|
+
|
36
|
+
@acl = nil
|
37
|
+
|
38
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
39
|
+
opt.on("-a", "--acl=ACL", "Options: #{AVAILABLE_ACLS.join ', '}") {|acl|
|
40
|
+
@acl = acl.to_sym
|
41
|
+
}
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def run s3, bucket, key, file, args
|
46
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
47
|
+
|
48
|
+
begin
|
49
|
+
params = {}
|
50
|
+
if @acl
|
51
|
+
raise WrongUsage.new(nil, "Invalid ACL `#{@acl}'. Should be any of #{AVAILABLE_ACLS.join ', '}") if not AVAILABLE_ACLS.include? @acl
|
52
|
+
params.merge!({:acl => @acl})
|
53
|
+
end
|
54
|
+
|
55
|
+
s3.buckets.create bucket, params
|
56
|
+
rescue AWS::S3::Errors::BucketAlreadyExists => exc
|
57
|
+
raise FailureFeedback.new("Bucket `#{bucket}' already exists")
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
class DeleteBucket < CmdParse::Command
|
63
|
+
attr_accessor :force
|
64
|
+
|
65
|
+
def initialize
|
66
|
+
super 'deletebucket', false, false
|
67
|
+
|
68
|
+
@short_desc = "Remove a bucket from your account"
|
69
|
+
|
70
|
+
@force = false
|
71
|
+
|
72
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
73
|
+
opt.on("-f", "--force", "Clean the bucket then deletes it") {|f|
|
74
|
+
@force = f
|
75
|
+
}
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def run s3, bucket, key, file, args
|
80
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
81
|
+
|
82
|
+
# Getting the bucket
|
83
|
+
bucket_obj = s3.buckets[bucket]
|
84
|
+
|
85
|
+
# Do not kill buckets with content unless explicitly asked
|
86
|
+
if not @force and bucket_obj.objects.count > 0
|
87
|
+
raise FailureFeedback.new("Cowardly refusing to remove non-empty bucket `#{bucket}'. Try with -f.")
|
88
|
+
end
|
89
|
+
|
90
|
+
bucket_obj.delete!
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
class List < CmdParse::Command
|
95
|
+
attr_accessor :max_entries
|
96
|
+
|
97
|
+
def initialize
|
98
|
+
super 'list', false, false
|
99
|
+
|
100
|
+
@short_desc = "List items filed under a given bucket"
|
101
|
+
|
102
|
+
@max_entries = 0
|
103
|
+
|
104
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
105
|
+
opt.on("-m", "--max-entries=NUM", "Limit the number of entries to output") {|m|
|
106
|
+
@max_entries = m
|
107
|
+
}
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def run s3, bucket, key, file, args
|
112
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
113
|
+
|
114
|
+
collection = s3.buckets[bucket].objects.with_prefix(key || "")
|
115
|
+
|
116
|
+
if @max_entries > 0
|
117
|
+
collection = collection.page(:per_page => max = @max_entries)
|
118
|
+
end
|
119
|
+
|
120
|
+
collection.each {|object|
|
121
|
+
puts "#{object.key}\t#{object.content_length}\t#{object.last_modified}"
|
122
|
+
}
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
class Delete < CmdParse::Command
|
127
|
+
def initialize
|
128
|
+
super 'delete', false, false, false
|
129
|
+
|
130
|
+
@short_desc = "Delete a key from a bucket"
|
131
|
+
end
|
132
|
+
|
133
|
+
def run s3, bucket, key, file, args
|
134
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
135
|
+
raise WrongUsage.new(nil, "You need to inform a key") if not key
|
136
|
+
s3.buckets[bucket].objects[key].delete
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
class Url < CmdParse::Command
|
141
|
+
attr_accessor :method
|
142
|
+
attr_accessor :secure
|
143
|
+
|
144
|
+
def initialize
|
145
|
+
super 'url', false, false
|
146
|
+
|
147
|
+
@short_desc = "Generates a url pointing to the given key"
|
148
|
+
@method = 'read'
|
149
|
+
@secure = true
|
150
|
+
@expires_in = false
|
151
|
+
|
152
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
153
|
+
opt.on("-m", "--method", "Options: #{AVAILABLE_METHODS.join ', '}") {|m|
|
154
|
+
@method = m
|
155
|
+
}
|
156
|
+
|
157
|
+
opt.on("--no-ssl", "Generate an HTTP link, no HTTPS") {
|
158
|
+
@secure = false
|
159
|
+
}
|
160
|
+
|
161
|
+
opt.on("--expires-in=EXPR", "How long the link takes to expire. Format: <# of seconds> | [#d|#h|#m|#s]") { |expr|
|
162
|
+
val = 0
|
163
|
+
expr.scan /(\d+\w)/ do |track|
|
164
|
+
_, num, what = /(\d+)(\w)/.match(track[0]).to_a
|
165
|
+
num = num.to_i
|
166
|
+
|
167
|
+
case what
|
168
|
+
when "d"; val += num * 86400
|
169
|
+
when "h"; val += num * 3600
|
170
|
+
when "m"; val += num * 60
|
171
|
+
when "s"; val += num
|
172
|
+
end
|
173
|
+
end
|
174
|
+
@expires_in = val > 0 ? val : expr.to_i
|
175
|
+
}
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
def run s3, bucket, key, file, args
|
180
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
181
|
+
raise WrongUsage.new(nil, "You need to inform a key") if not key
|
182
|
+
raise WrongUsage.new(nil, "Unknown method #{@method}") unless AVAILABLE_METHODS.include? @method
|
183
|
+
|
184
|
+
opts = {}
|
185
|
+
opts.merge!({:secure => @secure})
|
186
|
+
opts.merge!({:expires => @expires_in}) if @expires_in
|
187
|
+
puts (s3.buckets[bucket].objects[key].url_for @method.to_sym, opts).to_s
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
class Put < CmdParse::Command
|
192
|
+
def initialize
|
193
|
+
super 'put', false, false
|
194
|
+
|
195
|
+
@short_desc = 'Upload a file to a bucket under a certain prefix'
|
196
|
+
end
|
197
|
+
|
198
|
+
def run s3, bucket, key, file, args
|
199
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
200
|
+
raise WrongUsage.new(nil, "You need to inform a file") if not file
|
201
|
+
|
202
|
+
name = S3Ranger.safe_join [key, File.basename(file)]
|
203
|
+
s3.buckets[bucket].objects[name].write Pathname.new(file)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
class Get < CmdParse::Command
|
208
|
+
def initialize
|
209
|
+
super 'get', false, false
|
210
|
+
@short_desc = "Retrieve an object and save to the specified file"
|
211
|
+
end
|
212
|
+
|
213
|
+
def run s3, bucket, key, file, args
|
214
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
215
|
+
raise WrongUsage.new(nil, "You need to inform a key") if not key
|
216
|
+
raise WrongUsage.new(nil, "You need to inform a file") if not file
|
217
|
+
|
218
|
+
# Saving the content to be downloaded to the current directory if the
|
219
|
+
# destination is a directory
|
220
|
+
path = File.absolute_path file
|
221
|
+
path = S3Ranger.safe_join [path, File.basename(key)] if File.directory? path
|
222
|
+
File.open(path, 'wb') do |f|
|
223
|
+
s3.buckets[bucket].objects[key].read do |chunk| f.write(chunk) end
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
class Sync < CmdParse::Command
|
229
|
+
attr_accessor :s3
|
230
|
+
attr_accessor :exclude
|
231
|
+
attr_accessor :keep
|
232
|
+
attr_accessor :dry_run
|
233
|
+
attr_accessor :verbose
|
234
|
+
|
235
|
+
def initialize
|
236
|
+
super 'sync', false, false
|
237
|
+
|
238
|
+
@short_desc = "Synchronize an S3 and a local folder"
|
239
|
+
@s3 = nil
|
240
|
+
@exclude = nil
|
241
|
+
@keep = false
|
242
|
+
@dry_run = false
|
243
|
+
@verbose = false
|
244
|
+
|
245
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
246
|
+
opt.on("-x EXPR", "--exclude=EXPR", "") {|v|
|
247
|
+
@exclude = v
|
248
|
+
}
|
249
|
+
|
250
|
+
opt.on("-k", "--keep", "Keep files even if they don't exist in source") {
|
251
|
+
@keep = true
|
252
|
+
}
|
253
|
+
|
254
|
+
opt.on("-d", "--dry-run", "Do not download or exclude anything, just show what was planned. Implies `verbose`.") {
|
255
|
+
@dry_run = true
|
256
|
+
@verbose = true
|
257
|
+
}
|
258
|
+
|
259
|
+
opt.on("-v", "--verbose", "Show file names") {
|
260
|
+
@verbose = true
|
261
|
+
}
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
def run s3, bucket, key, file, args
|
266
|
+
@s3 = s3
|
267
|
+
cmd = SyncCommand.new self, *args
|
268
|
+
cmd.run
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
def run conf
|
273
|
+
cmd = CmdParse::CommandParser.new true
|
274
|
+
cmd.program_version = S3Ranger::VERSION
|
275
|
+
|
276
|
+
cmd.options = CmdParse::OptionParserWrapper.new do |opt|
|
277
|
+
opt.separator "Global options:"
|
278
|
+
end
|
279
|
+
|
280
|
+
cmd.main_command.short_desc = 'Tool belt for managing your S3 buckets'
|
281
|
+
cmd.main_command.description = [] \
|
282
|
+
<< "Below you have a list of commands will allow you to manage your content" \
|
283
|
+
<< "stored in S3 buckets. For more information on each command, you can always" \
|
284
|
+
<< "use the `--help' parameter, just like this:" \
|
285
|
+
<< "" \
|
286
|
+
<< " $ #{$0} sync --help" \
|
287
|
+
|
288
|
+
# Commands used more often
|
289
|
+
cmd.add_command List.new
|
290
|
+
cmd.add_command Delete.new
|
291
|
+
cmd.add_command Url.new
|
292
|
+
cmd.add_command Put.new
|
293
|
+
cmd.add_command Get.new
|
294
|
+
cmd.add_command Sync.new
|
295
|
+
|
296
|
+
# Bucket related options
|
297
|
+
cmd.add_command ListBuckets.new
|
298
|
+
cmd.add_command CreateBucket.new
|
299
|
+
cmd.add_command DeleteBucket.new
|
300
|
+
|
301
|
+
|
302
|
+
# Built-in commands
|
303
|
+
cmd.add_command CmdParse::HelpCommand.new
|
304
|
+
cmd.add_command CmdParse::VersionCommand.new
|
305
|
+
|
306
|
+
# Defining the `execute` method as a closure, so we can forward the
|
307
|
+
# arguments needed to run the instance of the chosen command.
|
308
|
+
CmdParse::Command.class_eval do
|
309
|
+
define_method :execute, lambda { |args|
|
310
|
+
|
311
|
+
# Connecting to amazon
|
312
|
+
s3 = AWS::S3.new(
|
313
|
+
:access_key_id => conf[:AWS_ACCESS_KEY_ID],
|
314
|
+
:secret_access_key => conf[:AWS_SECRET_ACCESS_KEY],
|
315
|
+
)
|
316
|
+
|
317
|
+
# From the command line
|
318
|
+
key, file = args
|
319
|
+
|
320
|
+
# Parsing the bucket name
|
321
|
+
bucket = nil
|
322
|
+
bucket, key = key.split(':') if key
|
323
|
+
|
324
|
+
# Running our custom method inside of the command class, taking care
|
325
|
+
# of the common errors here, saving duplications in each command;
|
326
|
+
begin
|
327
|
+
run s3, bucket, key, file, args
|
328
|
+
rescue AWS::S3::Errors::AccessDenied
|
329
|
+
raise FailureFeedback.new("Access Denied")
|
330
|
+
rescue AWS::S3::Errors::NoSuchBucket
|
331
|
+
raise FailureFeedback.new("There's no bucket named `#{bucket}'")
|
332
|
+
rescue AWS::S3::Errors::NoSuchKey
|
333
|
+
raise FailureFeedback.new("There's no key named `#{key}' in the bucket `#{bucket}'")
|
334
|
+
end
|
335
|
+
}
|
336
|
+
end
|
337
|
+
|
338
|
+
cmd.parse
|
339
|
+
end
|
340
|
+
|
341
|
+
module_function :run
|
342
|
+
|
343
|
+
end
|
344
|
+
end
|
data/lib/s3ranger/sync.rb
CHANGED
@@ -138,19 +138,19 @@ module S3Ranger
|
|
138
138
|
# Removing the items matching the exclude pattern if requested
|
139
139
|
to_add.select! { |e|
|
140
140
|
begin
|
141
|
-
(e.path =~ /#{@args
|
141
|
+
(e.path =~ /#{@args.exclude}/).nil?
|
142
142
|
rescue RegexpError => exc
|
143
143
|
raise WrongUsage.new nil, exc.message
|
144
144
|
end
|
145
|
-
} if @args
|
145
|
+
} if @args.exclude
|
146
146
|
|
147
147
|
# Calling the methods that perform the actual IO
|
148
148
|
if source.local?
|
149
149
|
upload_files destination, to_add
|
150
|
-
remove_files destination, to_remove unless @args
|
150
|
+
remove_files destination, to_remove unless @args.keep
|
151
151
|
else
|
152
152
|
download_files destination, source, to_add
|
153
|
-
remove_local_files destination, source, to_remove unless @args
|
153
|
+
remove_local_files destination, source, to_remove unless @args.keep
|
154
154
|
end
|
155
155
|
end
|
156
156
|
|
@@ -246,7 +246,7 @@ module S3Ranger
|
|
246
246
|
begin
|
247
247
|
dir = location.path
|
248
248
|
dir += '/' if not (dir.empty? or dir.end_with? '/')
|
249
|
-
@args
|
249
|
+
@args.s3.buckets[location.bucket].objects.with_prefix(dir || "").to_a.collect {|obj|
|
250
250
|
Node.new location.path, obj.key, obj.content_length
|
251
251
|
}
|
252
252
|
rescue AWS::S3::Errors::NoSuchBucket
|
@@ -272,13 +272,13 @@ module S3Ranger
|
|
272
272
|
|
273
273
|
def upload_files remote, list
|
274
274
|
list.each do |e|
|
275
|
-
if @args
|
275
|
+
if @args.verbose
|
276
276
|
puts " + #{e.full} => #{remote}#{e.path}"
|
277
277
|
end
|
278
278
|
|
279
|
-
unless @args
|
279
|
+
unless @args.dry_run
|
280
280
|
if File.file? e.path
|
281
|
-
@args
|
281
|
+
@args.s3.buckets[remote.bucket].objects[e.path].write Pathname.new e.path
|
282
282
|
end
|
283
283
|
end
|
284
284
|
end
|
@@ -286,14 +286,14 @@ module S3Ranger
|
|
286
286
|
|
287
287
|
def remove_files remote, list
|
288
288
|
|
289
|
-
if @args
|
289
|
+
if @args.verbose
|
290
290
|
list.each {|e|
|
291
291
|
puts " - #{remote}#{e.path}"
|
292
292
|
}
|
293
293
|
end
|
294
294
|
|
295
|
-
unless @args
|
296
|
-
@args
|
295
|
+
unless @args.dry_run
|
296
|
+
@args.s3.buckets[remote.bucket].objects.delete_if { |obj| list.include? obj.key }
|
297
297
|
end
|
298
298
|
end
|
299
299
|
|
@@ -301,12 +301,12 @@ module S3Ranger
|
|
301
301
|
list.each {|e|
|
302
302
|
path = File.join destination.path, e.path
|
303
303
|
|
304
|
-
if @args
|
304
|
+
if @args.verbose
|
305
305
|
puts " + #{source}#{e.path} => #{path}"
|
306
306
|
end
|
307
307
|
|
308
|
-
unless @args
|
309
|
-
obj = @args
|
308
|
+
unless @args.dry_run
|
309
|
+
obj = @args.s3.buckets[source.bucket].objects[e.path]
|
310
310
|
|
311
311
|
# Making sure this new file will have a safe shelter
|
312
312
|
FileUtils.mkdir_p File.dirname(path)
|
@@ -325,11 +325,11 @@ module S3Ranger
|
|
325
325
|
list.each {|e|
|
326
326
|
path = File.join destination.path, e.path
|
327
327
|
|
328
|
-
if @args
|
328
|
+
if @args.verbose
|
329
329
|
puts " * #{e.path} => #{path}"
|
330
330
|
end
|
331
331
|
|
332
|
-
unless @args
|
332
|
+
unless @args.dry_run
|
333
333
|
FileUtils.rm_rf path
|
334
334
|
end
|
335
335
|
}
|
data/lib/s3ranger/util.rb
CHANGED
@@ -2,4 +2,13 @@ module S3Ranger
|
|
2
2
|
def S3Ranger.safe_join(parts)
|
3
3
|
File.join *(parts.select {|v| !v.nil? && !v.empty? })
|
4
4
|
end
|
5
|
+
|
6
|
+
# class Object
|
7
|
+
# # note that this method is already defined in Ruby 1.9
|
8
|
+
# def define_singleton_method(name, callable = nil, &block)
|
9
|
+
# block ||= callable
|
10
|
+
# metaclass = class << self; self; end
|
11
|
+
# metaclass.send(:define_method, name, block)
|
12
|
+
# end
|
13
|
+
# end
|
5
14
|
end
|
data/lib/s3ranger/version.rb
CHANGED
data/s3ranger.gemspec
CHANGED
data/spec/main_spec.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3ranger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-08-
|
12
|
+
date: 2013-08-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk
|
@@ -27,6 +27,22 @@ dependencies:
|
|
27
27
|
- - ! '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: cmdparse
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
30
46
|
- !ruby/object:Gem::Dependency
|
31
47
|
name: debugger
|
32
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -125,8 +141,7 @@ files:
|
|
125
141
|
- Rakefile
|
126
142
|
- bin/s3ranger
|
127
143
|
- lib/s3ranger.rb
|
128
|
-
- lib/s3ranger/
|
129
|
-
- lib/s3ranger/commands.rb
|
144
|
+
- lib/s3ranger/cli.rb
|
130
145
|
- lib/s3ranger/config.rb
|
131
146
|
- lib/s3ranger/exceptions.rb
|
132
147
|
- lib/s3ranger/sync.rb
|
@@ -153,7 +168,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
153
168
|
version: '0'
|
154
169
|
segments:
|
155
170
|
- 0
|
156
|
-
hash:
|
171
|
+
hash: -223614671279918114
|
157
172
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
158
173
|
none: false
|
159
174
|
requirements:
|
@@ -162,7 +177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
162
177
|
version: '0'
|
163
178
|
segments:
|
164
179
|
- 0
|
165
|
-
hash:
|
180
|
+
hash: -223614671279918114
|
166
181
|
requirements: []
|
167
182
|
rubyforge_project:
|
168
183
|
rubygems_version: 1.8.24
|
data/lib/s3ranger/cmd.rb
DELETED
@@ -1,112 +0,0 @@
|
|
1
|
-
# (c) 2013 Lincoln de Sousa <lincoln@clarete.li>
|
2
|
-
# (c) 2007 s3sync.net
|
3
|
-
#
|
4
|
-
# This software code is made available "AS IS" without warranties of any
|
5
|
-
# kind. You may copy, display, modify and redistribute the software
|
6
|
-
# code either by itself or as incorporated into your code; provided that
|
7
|
-
# you do not remove any proprietary notices. Your use of this software
|
8
|
-
# code is at your own risk and you waive any claim against the author
|
9
|
-
# with respect to your use of this software code.
|
10
|
-
|
11
|
-
require 'getoptlong'
|
12
|
-
require 's3ranger/commands'
|
13
|
-
require 's3ranger/util'
|
14
|
-
|
15
|
-
module S3Ranger
|
16
|
-
|
17
|
-
class Cmd
|
18
|
-
|
19
|
-
def initialize(conf = conf)
|
20
|
-
# The chain that initializes our command and find the right action
|
21
|
-
options, command, bucket, key, file = read_info_from_args(parse_args())
|
22
|
-
|
23
|
-
# Finding the right command to run
|
24
|
-
(cmd = find_cmd(command)) || (raise WrongUsage.new(nil, "Command `#{command}' does not exist"))
|
25
|
-
|
26
|
-
# Now that we're sure we have things to do, we need to connect to amazon
|
27
|
-
s3 = AWS::S3.new(
|
28
|
-
:access_key_id => conf[:AWS_ACCESS_KEY_ID],
|
29
|
-
:secret_access_key => conf[:AWS_SECRET_ACCESS_KEY],
|
30
|
-
)
|
31
|
-
|
32
|
-
# Calling the actuall command
|
33
|
-
cmd.call({
|
34
|
-
:options => options,
|
35
|
-
:s3 => s3,
|
36
|
-
:bucket => bucket,
|
37
|
-
:key => key,
|
38
|
-
:file => file,
|
39
|
-
})
|
40
|
-
end
|
41
|
-
|
42
|
-
def find_cmd name
|
43
|
-
sym = "_cmd_#{name}".to_sym
|
44
|
-
return nil unless Commands.public_methods.include? sym
|
45
|
-
return Commands.method sym
|
46
|
-
end
|
47
|
-
|
48
|
-
def parse_args
|
49
|
-
options = Hash.new
|
50
|
-
|
51
|
-
args = [
|
52
|
-
['--help', '-h', GetoptLong::NO_ARGUMENT],
|
53
|
-
['--force', '-f', GetoptLong::NO_ARGUMENT],
|
54
|
-
['--acl', '-a', GetoptLong::REQUIRED_ARGUMENT],
|
55
|
-
['--method', '-m', GetoptLong::REQUIRED_ARGUMENT],
|
56
|
-
['--exclude', '-e', GetoptLong::REQUIRED_ARGUMENT],
|
57
|
-
['--keep', '-k', GetoptLong::NO_ARGUMENT],
|
58
|
-
['--dry-run', '-d', GetoptLong::NO_ARGUMENT],
|
59
|
-
['--verbose', '-v', GetoptLong::NO_ARGUMENT],
|
60
|
-
['--no-ssl', GetoptLong::NO_ARGUMENT],
|
61
|
-
['--expires-in', GetoptLong::REQUIRED_ARGUMENT],
|
62
|
-
]
|
63
|
-
|
64
|
-
begin
|
65
|
-
GetoptLong.new(*args).each {|opt, arg| options[opt] = (arg || true)}
|
66
|
-
rescue StandardError => exc
|
67
|
-
raise WrongUsage.new nil, exc.message
|
68
|
-
end
|
69
|
-
|
70
|
-
# Let's just show the help to the user
|
71
|
-
raise WrongUsage.new(0, nil) if options['--help']
|
72
|
-
|
73
|
-
# Returning the options to the next level
|
74
|
-
options
|
75
|
-
end
|
76
|
-
|
77
|
-
def read_info_from_args(options)
|
78
|
-
# Parsing expre date
|
79
|
-
if options['--expires-in'] =~ /d|h|m|s/
|
80
|
-
|
81
|
-
val = 0
|
82
|
-
|
83
|
-
options['--expires-in'].scan(/(\d+\w)/) do |track|
|
84
|
-
_, num, what = /(\d+)(\w)/.match(track[0]).to_a
|
85
|
-
num = num.to_i
|
86
|
-
|
87
|
-
case what
|
88
|
-
when "d"; val += num * 86400
|
89
|
-
when "h"; val += num * 3600
|
90
|
-
when "m"; val += num * 60
|
91
|
-
when "s"; val += num
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
options['--expires-in'] = val
|
96
|
-
end
|
97
|
-
|
98
|
-
# Reading what to do from the user
|
99
|
-
command = ARGV.shift
|
100
|
-
raise WrongUsage.new(nil, "Need a command (eg.: `list', `listbuckets', etc)") if not command
|
101
|
-
|
102
|
-
key, file = ARGV
|
103
|
-
|
104
|
-
# Parsing the bucket name
|
105
|
-
bucket = nil
|
106
|
-
bucket, key = key.split(':') if key
|
107
|
-
|
108
|
-
# Returning things we need in the next level
|
109
|
-
[options, command, bucket, key, file]
|
110
|
-
end
|
111
|
-
end
|
112
|
-
end
|
data/lib/s3ranger/commands.rb
DELETED
@@ -1,114 +0,0 @@
|
|
1
|
-
require 's3ranger/exceptions'
|
2
|
-
require 's3ranger/sync'
|
3
|
-
require 'aws/s3'
|
4
|
-
|
5
|
-
|
6
|
-
module Commands
|
7
|
-
|
8
|
-
include S3Ranger
|
9
|
-
|
10
|
-
AVAILABLE_ACLS = [:public_read, :public_read_write, :private]
|
11
|
-
|
12
|
-
AVAILABLE_METHODS = ['read', 'get', 'put', 'write', 'delete']
|
13
|
-
|
14
|
-
def Commands._cmd_listbuckets args
|
15
|
-
args[:s3].buckets.each do |bkt|
|
16
|
-
puts "#{bkt.name}"
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def Commands._cmd_createbucket args
|
21
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
22
|
-
|
23
|
-
begin
|
24
|
-
params = {}
|
25
|
-
if acl = args[:options]['--acl']
|
26
|
-
raise WrongUsage.new(nil, "Invalid ACL. Should be any of #{EXISTING_ACLS.join ', '}") if not AVAILABLE_ACLS.include? acl
|
27
|
-
params.merge!({:acl => acl.to_sym})
|
28
|
-
end
|
29
|
-
|
30
|
-
args[:s3].buckets.create args[:bucket], params
|
31
|
-
rescue AWS::S3::Errors::BucketAlreadyExists => exc
|
32
|
-
raise FailureFeedback.new("Bucket `#{bucket}' already exists")
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def Commands._cmd_deletebucket args
|
37
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
38
|
-
|
39
|
-
# Getting the bucket
|
40
|
-
bucket_obj = args[:s3].buckets[args[:bucket]]
|
41
|
-
|
42
|
-
# Do not kill buckets with content unless explicitly asked
|
43
|
-
if not args[:options]['--force'] and bucket_obj.objects.count > 0
|
44
|
-
raise FailureFeedback.new("Cowardly refusing to remove non-empty bucket `#{bucket}'. Try with -f.")
|
45
|
-
end
|
46
|
-
|
47
|
-
begin
|
48
|
-
bucket_obj.delete!
|
49
|
-
rescue AWS::S3::Errors::AccessDenied => exc
|
50
|
-
raise FailureFeedback.new("Access Denied")
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def Commands._cmd_list args
|
55
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
56
|
-
args[:s3].buckets[args[:bucket]].objects.with_prefix(args[:key] || "").each do |object|
|
57
|
-
puts "#{object.key}\t#{object.content_length}\t#{object.last_modified}"
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
def Commands._cmd_delete args
|
62
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
63
|
-
raise WrongUsage.new(nil, "You need to inform a key") if not args[:key]
|
64
|
-
args[:s3].buckets[args[:bucket]].objects[args[:key]].delete
|
65
|
-
end
|
66
|
-
|
67
|
-
def Commands._cmd_url args
|
68
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
69
|
-
raise WrongUsage.new(nil, "You need to inform a key") if not args[:key]
|
70
|
-
|
71
|
-
method = args[:options]['--method'] || 'read'
|
72
|
-
raise WrongUsage.new(nil, "") unless AVAILABLE_METHODS.include? method
|
73
|
-
|
74
|
-
opts = {}
|
75
|
-
opts.merge!({:secure => args[:options]["--no-ssl"].nil?})
|
76
|
-
opts.merge!({:expires => args[:options]["--expires-in"]}) if args[:options]["--expires-in"]
|
77
|
-
p (args[:s3].buckets[args[:bucket]].objects[args[:key]].url_for method.to_sym, opts).to_s
|
78
|
-
end
|
79
|
-
|
80
|
-
def Commands._cmd_put args
|
81
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
82
|
-
raise WrongUsage.new(nil, "You need to inform a file") if not args[:file]
|
83
|
-
|
84
|
-
# key + file name
|
85
|
-
name = S3Ranger.safe_join [args[:key], File.basename(args[:file])]
|
86
|
-
args[:s3].buckets[args[:bucket]].objects[name].write Pathname.new(args[:file])
|
87
|
-
end
|
88
|
-
|
89
|
-
def Commands._cmd_get args
|
90
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
91
|
-
raise WrongUsage.new(nil, "You need to inform a key") if not args[:key]
|
92
|
-
raise WrongUsage.new(nil, "You need to inform a file") if not args[:file]
|
93
|
-
|
94
|
-
# Saving the content to be downloaded to the current directory if the
|
95
|
-
# destination is a directory
|
96
|
-
path = File.absolute_path args[:file]
|
97
|
-
path = S3Ranger.safe_join [path, File.basename(args[:key])] if File.directory? path
|
98
|
-
|
99
|
-
File.open(path, 'wb') do |f|
|
100
|
-
begin
|
101
|
-
args[:s3].buckets[args[:bucket]].objects[args[:key]].read do |chunk| f.write(chunk) end
|
102
|
-
rescue AWS::S3::Errors::NoSuchBucket
|
103
|
-
raise FailureFeedback.new("There's no bucket named `#{args[:bucket]}'")
|
104
|
-
rescue AWS::S3::Errors::NoSuchKey
|
105
|
-
raise FailureFeedback.new("There's no key named `#{args[:key]}' in the bucket `#{args[:bucket]}'")
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
|
-
def Commands._cmd_sync args
|
111
|
-
cmd = SyncCommand.new args, *ARGV
|
112
|
-
cmd.run
|
113
|
-
end
|
114
|
-
end
|