s3ranger 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +4 -2
- data/bin/s3ranger +3 -69
- data/lib/s3ranger/cli.rb +344 -0
- data/lib/s3ranger/sync.rb +16 -16
- data/lib/s3ranger/util.rb +9 -0
- data/lib/s3ranger/version.rb +1 -1
- data/s3ranger.gemspec +1 -0
- data/spec/main_spec.rb +1 -2
- metadata +21 -6
- data/lib/s3ranger/cmd.rb +0 -112
- data/lib/s3ranger/commands.rb +0 -114
data/Gemfile.lock
CHANGED
@@ -1,16 +1,18 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
s3ranger (0.
|
4
|
+
s3ranger (0.2.0)
|
5
5
|
aws-sdk
|
6
|
+
cmdparse
|
6
7
|
|
7
8
|
GEM
|
8
9
|
remote: https://rubygems.org/
|
9
10
|
specs:
|
10
|
-
aws-sdk (1.
|
11
|
+
aws-sdk (1.16.0)
|
11
12
|
json (~> 1.4)
|
12
13
|
nokogiri (< 1.6.0)
|
13
14
|
uuidtools (~> 2.1)
|
15
|
+
cmdparse (2.0.5)
|
14
16
|
columnize (0.3.6)
|
15
17
|
debugger (1.6.1)
|
16
18
|
columnize (>= 0.3.1)
|
data/bin/s3ranger
CHANGED
@@ -4,7 +4,7 @@ $:.unshift(File.dirname(__FILE__) + '/../lib') unless $:.include?(File.dirname(_
|
|
4
4
|
|
5
5
|
require "s3ranger/exceptions"
|
6
6
|
require "s3ranger/config"
|
7
|
-
require "s3ranger/
|
7
|
+
require "s3ranger/cli"
|
8
8
|
|
9
9
|
conf = S3Ranger::Config.new
|
10
10
|
|
@@ -24,82 +24,16 @@ rescue S3Ranger::NoConfigFound => exc
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
-
|
28
27
|
# Step aside, the star of this show is here. Let's try to create the
|
29
28
|
# environment to run the requested command. And feed the user back if
|
30
29
|
# information needed was not enough
|
31
30
|
begin
|
32
|
-
S3Ranger::
|
33
|
-
|
31
|
+
S3Ranger::CLI::run conf
|
34
32
|
rescue S3Ranger::FailureFeedback => exc
|
35
33
|
$stderr.puts exc.message
|
36
34
|
exit 1
|
37
|
-
|
38
35
|
rescue S3Ranger::WrongUsage => exc
|
39
36
|
name = $0.split('/').last
|
40
|
-
|
41
|
-
$stderr.puts <<"ENDUSAGE"
|
42
|
-
Usage: #{name} [options] <command> [arg(s)]
|
43
|
-
|
44
|
-
Global Options:
|
45
|
-
-h, --help
|
46
|
-
-v, --verbose
|
47
|
-
-n, --dryrun
|
48
|
-
-d, --debug
|
49
|
-
--progress
|
50
|
-
|
51
|
-
#{name} listbuckets
|
52
|
-
List all available buckets
|
53
|
-
|
54
|
-
#{name} createbucket <bucket>
|
55
|
-
Creates a new bucket
|
56
|
-
|
57
|
-
Options:
|
58
|
-
|
59
|
-
-a <ACL>, --acl=(private|public_read|public_read_write)
|
60
|
-
|
61
|
-
#{name} deletebucket [options] <bucket>
|
62
|
-
Removes an existing bucket
|
63
|
-
|
64
|
-
Options:
|
65
|
-
-f, --force Deletes non-empty buckets (BE CAREFUL)
|
66
|
-
|
67
|
-
#{name} list <bucket>[:prefix]
|
68
|
-
List content inside of bucket
|
69
|
-
|
70
|
-
if `prefix' is present, only content under `prefix' will be listed.
|
71
|
-
|
72
|
-
#{name} delete <bucket>:<key>
|
73
|
-
Removes `key` from `bucket`
|
74
|
-
|
75
|
-
#{name} url [options] <bucket>:<key>
|
76
|
-
Generates a presigned URL for an operation on the object named `key' found on
|
77
|
-
`bucket'.
|
78
|
-
|
79
|
-
Options:
|
80
|
-
--no-ssl
|
81
|
-
--expires-in=(<# of seconds> | [#d|#h|#m|#s])
|
82
|
-
|
83
|
-
#{name} get <bucket>:<key> <file>
|
84
|
-
Retrieves the remote `key` object from `bucket` and saves to the local path
|
85
|
-
specified in `file`
|
86
|
-
|
87
|
-
#{name} put <bucket>[:<key>] <file>
|
88
|
-
Uploads the file `file` to the `bucket` under the path `key`
|
89
|
-
|
90
|
-
#{name} sync <source> <destination>
|
91
|
-
|
92
|
-
One of source or destination must be remote and the other must be local,
|
93
|
-
where local points to a folder in the file system and remote conform to the
|
94
|
-
format `<bucket>[:<key>]`.
|
95
|
-
|
96
|
-
Options:
|
97
|
-
-e <pattern>, --exclude=<pattern>
|
98
|
-
-k, --keep
|
99
|
-
-d, --dry-run
|
100
|
-
|
101
|
-
ENDUSAGE
|
102
|
-
|
103
|
-
$stderr.puts "\nCurrent error:\n #{exc.msg}\n" if exc.msg
|
37
|
+
$stderr.puts "Error:\n #{exc.msg}\n" if exc.msg
|
104
38
|
exit exc.error_code
|
105
39
|
end
|
data/lib/s3ranger/cli.rb
ADDED
@@ -0,0 +1,344 @@
|
|
1
|
+
require 's3ranger/exceptions'
|
2
|
+
require 's3ranger/sync'
|
3
|
+
require 'aws/s3'
|
4
|
+
require 'cmdparse'
|
5
|
+
|
6
|
+
|
7
|
+
module S3Ranger
|
8
|
+
module CLI
|
9
|
+
|
10
|
+
AVAILABLE_ACLS = [:public_read, :public_read_write, :private]
|
11
|
+
|
12
|
+
AVAILABLE_METHODS = ['read', 'get', 'put', 'write', 'delete']
|
13
|
+
|
14
|
+
class ListBuckets < CmdParse::Command
|
15
|
+
def initialize
|
16
|
+
super 'listbuckets', false, false, false
|
17
|
+
|
18
|
+
@short_desc = "List all available buckets for your user"
|
19
|
+
end
|
20
|
+
|
21
|
+
def run s3, bucket, key, file, args
|
22
|
+
s3.buckets.each do |bkt|
|
23
|
+
puts "#{bkt.name}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
class CreateBucket < CmdParse::Command
|
29
|
+
attr_accessor :acl
|
30
|
+
|
31
|
+
def initialize
|
32
|
+
super 'createbucket', false, false
|
33
|
+
|
34
|
+
@short_desc = "Create a new bucket under your user account"
|
35
|
+
|
36
|
+
@acl = nil
|
37
|
+
|
38
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
39
|
+
opt.on("-a", "--acl=ACL", "Options: #{AVAILABLE_ACLS.join ', '}") {|acl|
|
40
|
+
@acl = acl.to_sym
|
41
|
+
}
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def run s3, bucket, key, file, args
|
46
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
47
|
+
|
48
|
+
begin
|
49
|
+
params = {}
|
50
|
+
if @acl
|
51
|
+
raise WrongUsage.new(nil, "Invalid ACL `#{@acl}'. Should be any of #{AVAILABLE_ACLS.join ', '}") if not AVAILABLE_ACLS.include? @acl
|
52
|
+
params.merge!({:acl => @acl})
|
53
|
+
end
|
54
|
+
|
55
|
+
s3.buckets.create bucket, params
|
56
|
+
rescue AWS::S3::Errors::BucketAlreadyExists => exc
|
57
|
+
raise FailureFeedback.new("Bucket `#{bucket}' already exists")
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
class DeleteBucket < CmdParse::Command
|
63
|
+
attr_accessor :force
|
64
|
+
|
65
|
+
def initialize
|
66
|
+
super 'deletebucket', false, false
|
67
|
+
|
68
|
+
@short_desc = "Remove a bucket from your account"
|
69
|
+
|
70
|
+
@force = false
|
71
|
+
|
72
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
73
|
+
opt.on("-f", "--force", "Clean the bucket then deletes it") {|f|
|
74
|
+
@force = f
|
75
|
+
}
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def run s3, bucket, key, file, args
|
80
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
81
|
+
|
82
|
+
# Getting the bucket
|
83
|
+
bucket_obj = s3.buckets[bucket]
|
84
|
+
|
85
|
+
# Do not kill buckets with content unless explicitly asked
|
86
|
+
if not @force and bucket_obj.objects.count > 0
|
87
|
+
raise FailureFeedback.new("Cowardly refusing to remove non-empty bucket `#{bucket}'. Try with -f.")
|
88
|
+
end
|
89
|
+
|
90
|
+
bucket_obj.delete!
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
class List < CmdParse::Command
|
95
|
+
attr_accessor :max_entries
|
96
|
+
|
97
|
+
def initialize
|
98
|
+
super 'list', false, false
|
99
|
+
|
100
|
+
@short_desc = "List items filed under a given bucket"
|
101
|
+
|
102
|
+
@max_entries = 0
|
103
|
+
|
104
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
105
|
+
opt.on("-m", "--max-entries=NUM", "Limit the number of entries to output") {|m|
|
106
|
+
@max_entries = m
|
107
|
+
}
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def run s3, bucket, key, file, args
|
112
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
113
|
+
|
114
|
+
collection = s3.buckets[bucket].objects.with_prefix(key || "")
|
115
|
+
|
116
|
+
if @max_entries > 0
|
117
|
+
collection = collection.page(:per_page => max = @max_entries)
|
118
|
+
end
|
119
|
+
|
120
|
+
collection.each {|object|
|
121
|
+
puts "#{object.key}\t#{object.content_length}\t#{object.last_modified}"
|
122
|
+
}
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
class Delete < CmdParse::Command
|
127
|
+
def initialize
|
128
|
+
super 'delete', false, false, false
|
129
|
+
|
130
|
+
@short_desc = "Delete a key from a bucket"
|
131
|
+
end
|
132
|
+
|
133
|
+
def run s3, bucket, key, file, args
|
134
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
135
|
+
raise WrongUsage.new(nil, "You need to inform a key") if not key
|
136
|
+
s3.buckets[bucket].objects[key].delete
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
class Url < CmdParse::Command
|
141
|
+
attr_accessor :method
|
142
|
+
attr_accessor :secure
|
143
|
+
|
144
|
+
def initialize
|
145
|
+
super 'url', false, false
|
146
|
+
|
147
|
+
@short_desc = "Generates a url pointing to the given key"
|
148
|
+
@method = 'read'
|
149
|
+
@secure = true
|
150
|
+
@expires_in = false
|
151
|
+
|
152
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
153
|
+
opt.on("-m", "--method", "Options: #{AVAILABLE_METHODS.join ', '}") {|m|
|
154
|
+
@method = m
|
155
|
+
}
|
156
|
+
|
157
|
+
opt.on("--no-ssl", "Generate an HTTP link, no HTTPS") {
|
158
|
+
@secure = false
|
159
|
+
}
|
160
|
+
|
161
|
+
opt.on("--expires-in=EXPR", "How long the link takes to expire. Format: <# of seconds> | [#d|#h|#m|#s]") { |expr|
|
162
|
+
val = 0
|
163
|
+
expr.scan /(\d+\w)/ do |track|
|
164
|
+
_, num, what = /(\d+)(\w)/.match(track[0]).to_a
|
165
|
+
num = num.to_i
|
166
|
+
|
167
|
+
case what
|
168
|
+
when "d"; val += num * 86400
|
169
|
+
when "h"; val += num * 3600
|
170
|
+
when "m"; val += num * 60
|
171
|
+
when "s"; val += num
|
172
|
+
end
|
173
|
+
end
|
174
|
+
@expires_in = val > 0 ? val : expr.to_i
|
175
|
+
}
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
def run s3, bucket, key, file, args
|
180
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
181
|
+
raise WrongUsage.new(nil, "You need to inform a key") if not key
|
182
|
+
raise WrongUsage.new(nil, "Unknown method #{@method}") unless AVAILABLE_METHODS.include? @method
|
183
|
+
|
184
|
+
opts = {}
|
185
|
+
opts.merge!({:secure => @secure})
|
186
|
+
opts.merge!({:expires => @expires_in}) if @expires_in
|
187
|
+
puts (s3.buckets[bucket].objects[key].url_for @method.to_sym, opts).to_s
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
class Put < CmdParse::Command
|
192
|
+
def initialize
|
193
|
+
super 'put', false, false
|
194
|
+
|
195
|
+
@short_desc = 'Upload a file to a bucket under a certain prefix'
|
196
|
+
end
|
197
|
+
|
198
|
+
def run s3, bucket, key, file, args
|
199
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
200
|
+
raise WrongUsage.new(nil, "You need to inform a file") if not file
|
201
|
+
|
202
|
+
name = S3Ranger.safe_join [key, File.basename(file)]
|
203
|
+
s3.buckets[bucket].objects[name].write Pathname.new(file)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
class Get < CmdParse::Command
|
208
|
+
def initialize
|
209
|
+
super 'get', false, false
|
210
|
+
@short_desc = "Retrieve an object and save to the specified file"
|
211
|
+
end
|
212
|
+
|
213
|
+
def run s3, bucket, key, file, args
|
214
|
+
raise WrongUsage.new(nil, "You need to inform a bucket") if not bucket
|
215
|
+
raise WrongUsage.new(nil, "You need to inform a key") if not key
|
216
|
+
raise WrongUsage.new(nil, "You need to inform a file") if not file
|
217
|
+
|
218
|
+
# Saving the content to be downloaded to the current directory if the
|
219
|
+
# destination is a directory
|
220
|
+
path = File.absolute_path file
|
221
|
+
path = S3Ranger.safe_join [path, File.basename(key)] if File.directory? path
|
222
|
+
File.open(path, 'wb') do |f|
|
223
|
+
s3.buckets[bucket].objects[key].read do |chunk| f.write(chunk) end
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
class Sync < CmdParse::Command
|
229
|
+
attr_accessor :s3
|
230
|
+
attr_accessor :exclude
|
231
|
+
attr_accessor :keep
|
232
|
+
attr_accessor :dry_run
|
233
|
+
attr_accessor :verbose
|
234
|
+
|
235
|
+
def initialize
|
236
|
+
super 'sync', false, false
|
237
|
+
|
238
|
+
@short_desc = "Synchronize an S3 and a local folder"
|
239
|
+
@s3 = nil
|
240
|
+
@exclude = nil
|
241
|
+
@keep = false
|
242
|
+
@dry_run = false
|
243
|
+
@verbose = false
|
244
|
+
|
245
|
+
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
246
|
+
opt.on("-x EXPR", "--exclude=EXPR", "") {|v|
|
247
|
+
@exclude = v
|
248
|
+
}
|
249
|
+
|
250
|
+
opt.on("-k", "--keep", "Keep files even if they don't exist in source") {
|
251
|
+
@keep = true
|
252
|
+
}
|
253
|
+
|
254
|
+
opt.on("-d", "--dry-run", "Do not download or exclude anything, just show what was planned. Implies `verbose`.") {
|
255
|
+
@dry_run = true
|
256
|
+
@verbose = true
|
257
|
+
}
|
258
|
+
|
259
|
+
opt.on("-v", "--verbose", "Show file names") {
|
260
|
+
@verbose = true
|
261
|
+
}
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
def run s3, bucket, key, file, args
|
266
|
+
@s3 = s3
|
267
|
+
cmd = SyncCommand.new self, *args
|
268
|
+
cmd.run
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
def run conf
|
273
|
+
cmd = CmdParse::CommandParser.new true
|
274
|
+
cmd.program_version = S3Ranger::VERSION
|
275
|
+
|
276
|
+
cmd.options = CmdParse::OptionParserWrapper.new do |opt|
|
277
|
+
opt.separator "Global options:"
|
278
|
+
end
|
279
|
+
|
280
|
+
cmd.main_command.short_desc = 'Tool belt for managing your S3 buckets'
|
281
|
+
cmd.main_command.description = [] \
|
282
|
+
<< "Below you have a list of commands will allow you to manage your content" \
|
283
|
+
<< "stored in S3 buckets. For more information on each command, you can always" \
|
284
|
+
<< "use the `--help' parameter, just like this:" \
|
285
|
+
<< "" \
|
286
|
+
<< " $ #{$0} sync --help" \
|
287
|
+
|
288
|
+
# Commands used more often
|
289
|
+
cmd.add_command List.new
|
290
|
+
cmd.add_command Delete.new
|
291
|
+
cmd.add_command Url.new
|
292
|
+
cmd.add_command Put.new
|
293
|
+
cmd.add_command Get.new
|
294
|
+
cmd.add_command Sync.new
|
295
|
+
|
296
|
+
# Bucket related options
|
297
|
+
cmd.add_command ListBuckets.new
|
298
|
+
cmd.add_command CreateBucket.new
|
299
|
+
cmd.add_command DeleteBucket.new
|
300
|
+
|
301
|
+
|
302
|
+
# Built-in commands
|
303
|
+
cmd.add_command CmdParse::HelpCommand.new
|
304
|
+
cmd.add_command CmdParse::VersionCommand.new
|
305
|
+
|
306
|
+
# Defining the `execute` method as a closure, so we can forward the
|
307
|
+
# arguments needed to run the instance of the chosen command.
|
308
|
+
CmdParse::Command.class_eval do
|
309
|
+
define_method :execute, lambda { |args|
|
310
|
+
|
311
|
+
# Connecting to amazon
|
312
|
+
s3 = AWS::S3.new(
|
313
|
+
:access_key_id => conf[:AWS_ACCESS_KEY_ID],
|
314
|
+
:secret_access_key => conf[:AWS_SECRET_ACCESS_KEY],
|
315
|
+
)
|
316
|
+
|
317
|
+
# From the command line
|
318
|
+
key, file = args
|
319
|
+
|
320
|
+
# Parsing the bucket name
|
321
|
+
bucket = nil
|
322
|
+
bucket, key = key.split(':') if key
|
323
|
+
|
324
|
+
# Running our custom method inside of the command class, taking care
|
325
|
+
# of the common errors here, saving duplications in each command;
|
326
|
+
begin
|
327
|
+
run s3, bucket, key, file, args
|
328
|
+
rescue AWS::S3::Errors::AccessDenied
|
329
|
+
raise FailureFeedback.new("Access Denied")
|
330
|
+
rescue AWS::S3::Errors::NoSuchBucket
|
331
|
+
raise FailureFeedback.new("There's no bucket named `#{bucket}'")
|
332
|
+
rescue AWS::S3::Errors::NoSuchKey
|
333
|
+
raise FailureFeedback.new("There's no key named `#{key}' in the bucket `#{bucket}'")
|
334
|
+
end
|
335
|
+
}
|
336
|
+
end
|
337
|
+
|
338
|
+
cmd.parse
|
339
|
+
end
|
340
|
+
|
341
|
+
module_function :run
|
342
|
+
|
343
|
+
end
|
344
|
+
end
|
data/lib/s3ranger/sync.rb
CHANGED
@@ -138,19 +138,19 @@ module S3Ranger
|
|
138
138
|
# Removing the items matching the exclude pattern if requested
|
139
139
|
to_add.select! { |e|
|
140
140
|
begin
|
141
|
-
(e.path =~ /#{@args
|
141
|
+
(e.path =~ /#{@args.exclude}/).nil?
|
142
142
|
rescue RegexpError => exc
|
143
143
|
raise WrongUsage.new nil, exc.message
|
144
144
|
end
|
145
|
-
} if @args
|
145
|
+
} if @args.exclude
|
146
146
|
|
147
147
|
# Calling the methods that perform the actual IO
|
148
148
|
if source.local?
|
149
149
|
upload_files destination, to_add
|
150
|
-
remove_files destination, to_remove unless @args
|
150
|
+
remove_files destination, to_remove unless @args.keep
|
151
151
|
else
|
152
152
|
download_files destination, source, to_add
|
153
|
-
remove_local_files destination, source, to_remove unless @args
|
153
|
+
remove_local_files destination, source, to_remove unless @args.keep
|
154
154
|
end
|
155
155
|
end
|
156
156
|
|
@@ -246,7 +246,7 @@ module S3Ranger
|
|
246
246
|
begin
|
247
247
|
dir = location.path
|
248
248
|
dir += '/' if not (dir.empty? or dir.end_with? '/')
|
249
|
-
@args
|
249
|
+
@args.s3.buckets[location.bucket].objects.with_prefix(dir || "").to_a.collect {|obj|
|
250
250
|
Node.new location.path, obj.key, obj.content_length
|
251
251
|
}
|
252
252
|
rescue AWS::S3::Errors::NoSuchBucket
|
@@ -272,13 +272,13 @@ module S3Ranger
|
|
272
272
|
|
273
273
|
def upload_files remote, list
|
274
274
|
list.each do |e|
|
275
|
-
if @args
|
275
|
+
if @args.verbose
|
276
276
|
puts " + #{e.full} => #{remote}#{e.path}"
|
277
277
|
end
|
278
278
|
|
279
|
-
unless @args
|
279
|
+
unless @args.dry_run
|
280
280
|
if File.file? e.path
|
281
|
-
@args
|
281
|
+
@args.s3.buckets[remote.bucket].objects[e.path].write Pathname.new e.path
|
282
282
|
end
|
283
283
|
end
|
284
284
|
end
|
@@ -286,14 +286,14 @@ module S3Ranger
|
|
286
286
|
|
287
287
|
def remove_files remote, list
|
288
288
|
|
289
|
-
if @args
|
289
|
+
if @args.verbose
|
290
290
|
list.each {|e|
|
291
291
|
puts " - #{remote}#{e.path}"
|
292
292
|
}
|
293
293
|
end
|
294
294
|
|
295
|
-
unless @args
|
296
|
-
@args
|
295
|
+
unless @args.dry_run
|
296
|
+
@args.s3.buckets[remote.bucket].objects.delete_if { |obj| list.include? obj.key }
|
297
297
|
end
|
298
298
|
end
|
299
299
|
|
@@ -301,12 +301,12 @@ module S3Ranger
|
|
301
301
|
list.each {|e|
|
302
302
|
path = File.join destination.path, e.path
|
303
303
|
|
304
|
-
if @args
|
304
|
+
if @args.verbose
|
305
305
|
puts " + #{source}#{e.path} => #{path}"
|
306
306
|
end
|
307
307
|
|
308
|
-
unless @args
|
309
|
-
obj = @args
|
308
|
+
unless @args.dry_run
|
309
|
+
obj = @args.s3.buckets[source.bucket].objects[e.path]
|
310
310
|
|
311
311
|
# Making sure this new file will have a safe shelter
|
312
312
|
FileUtils.mkdir_p File.dirname(path)
|
@@ -325,11 +325,11 @@ module S3Ranger
|
|
325
325
|
list.each {|e|
|
326
326
|
path = File.join destination.path, e.path
|
327
327
|
|
328
|
-
if @args
|
328
|
+
if @args.verbose
|
329
329
|
puts " * #{e.path} => #{path}"
|
330
330
|
end
|
331
331
|
|
332
|
-
unless @args
|
332
|
+
unless @args.dry_run
|
333
333
|
FileUtils.rm_rf path
|
334
334
|
end
|
335
335
|
}
|
data/lib/s3ranger/util.rb
CHANGED
@@ -2,4 +2,13 @@ module S3Ranger
|
|
2
2
|
def S3Ranger.safe_join(parts)
|
3
3
|
File.join *(parts.select {|v| !v.nil? && !v.empty? })
|
4
4
|
end
|
5
|
+
|
6
|
+
# class Object
|
7
|
+
# # note that this method is already defined in Ruby 1.9
|
8
|
+
# def define_singleton_method(name, callable = nil, &block)
|
9
|
+
# block ||= callable
|
10
|
+
# metaclass = class << self; self; end
|
11
|
+
# metaclass.send(:define_method, name, block)
|
12
|
+
# end
|
13
|
+
# end
|
5
14
|
end
|
data/lib/s3ranger/version.rb
CHANGED
data/s3ranger.gemspec
CHANGED
data/spec/main_spec.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3ranger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-08-
|
12
|
+
date: 2013-08-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk
|
@@ -27,6 +27,22 @@ dependencies:
|
|
27
27
|
- - ! '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: cmdparse
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
30
46
|
- !ruby/object:Gem::Dependency
|
31
47
|
name: debugger
|
32
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -125,8 +141,7 @@ files:
|
|
125
141
|
- Rakefile
|
126
142
|
- bin/s3ranger
|
127
143
|
- lib/s3ranger.rb
|
128
|
-
- lib/s3ranger/
|
129
|
-
- lib/s3ranger/commands.rb
|
144
|
+
- lib/s3ranger/cli.rb
|
130
145
|
- lib/s3ranger/config.rb
|
131
146
|
- lib/s3ranger/exceptions.rb
|
132
147
|
- lib/s3ranger/sync.rb
|
@@ -153,7 +168,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
153
168
|
version: '0'
|
154
169
|
segments:
|
155
170
|
- 0
|
156
|
-
hash:
|
171
|
+
hash: -223614671279918114
|
157
172
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
158
173
|
none: false
|
159
174
|
requirements:
|
@@ -162,7 +177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
162
177
|
version: '0'
|
163
178
|
segments:
|
164
179
|
- 0
|
165
|
-
hash:
|
180
|
+
hash: -223614671279918114
|
166
181
|
requirements: []
|
167
182
|
rubyforge_project:
|
168
183
|
rubygems_version: 1.8.24
|
data/lib/s3ranger/cmd.rb
DELETED
@@ -1,112 +0,0 @@
|
|
1
|
-
# (c) 2013 Lincoln de Sousa <lincoln@clarete.li>
|
2
|
-
# (c) 2007 s3sync.net
|
3
|
-
#
|
4
|
-
# This software code is made available "AS IS" without warranties of any
|
5
|
-
# kind. You may copy, display, modify and redistribute the software
|
6
|
-
# code either by itself or as incorporated into your code; provided that
|
7
|
-
# you do not remove any proprietary notices. Your use of this software
|
8
|
-
# code is at your own risk and you waive any claim against the author
|
9
|
-
# with respect to your use of this software code.
|
10
|
-
|
11
|
-
require 'getoptlong'
|
12
|
-
require 's3ranger/commands'
|
13
|
-
require 's3ranger/util'
|
14
|
-
|
15
|
-
module S3Ranger
|
16
|
-
|
17
|
-
class Cmd
|
18
|
-
|
19
|
-
def initialize(conf = conf)
|
20
|
-
# The chain that initializes our command and find the right action
|
21
|
-
options, command, bucket, key, file = read_info_from_args(parse_args())
|
22
|
-
|
23
|
-
# Finding the right command to run
|
24
|
-
(cmd = find_cmd(command)) || (raise WrongUsage.new(nil, "Command `#{command}' does not exist"))
|
25
|
-
|
26
|
-
# Now that we're sure we have things to do, we need to connect to amazon
|
27
|
-
s3 = AWS::S3.new(
|
28
|
-
:access_key_id => conf[:AWS_ACCESS_KEY_ID],
|
29
|
-
:secret_access_key => conf[:AWS_SECRET_ACCESS_KEY],
|
30
|
-
)
|
31
|
-
|
32
|
-
# Calling the actuall command
|
33
|
-
cmd.call({
|
34
|
-
:options => options,
|
35
|
-
:s3 => s3,
|
36
|
-
:bucket => bucket,
|
37
|
-
:key => key,
|
38
|
-
:file => file,
|
39
|
-
})
|
40
|
-
end
|
41
|
-
|
42
|
-
def find_cmd name
|
43
|
-
sym = "_cmd_#{name}".to_sym
|
44
|
-
return nil unless Commands.public_methods.include? sym
|
45
|
-
return Commands.method sym
|
46
|
-
end
|
47
|
-
|
48
|
-
def parse_args
|
49
|
-
options = Hash.new
|
50
|
-
|
51
|
-
args = [
|
52
|
-
['--help', '-h', GetoptLong::NO_ARGUMENT],
|
53
|
-
['--force', '-f', GetoptLong::NO_ARGUMENT],
|
54
|
-
['--acl', '-a', GetoptLong::REQUIRED_ARGUMENT],
|
55
|
-
['--method', '-m', GetoptLong::REQUIRED_ARGUMENT],
|
56
|
-
['--exclude', '-e', GetoptLong::REQUIRED_ARGUMENT],
|
57
|
-
['--keep', '-k', GetoptLong::NO_ARGUMENT],
|
58
|
-
['--dry-run', '-d', GetoptLong::NO_ARGUMENT],
|
59
|
-
['--verbose', '-v', GetoptLong::NO_ARGUMENT],
|
60
|
-
['--no-ssl', GetoptLong::NO_ARGUMENT],
|
61
|
-
['--expires-in', GetoptLong::REQUIRED_ARGUMENT],
|
62
|
-
]
|
63
|
-
|
64
|
-
begin
|
65
|
-
GetoptLong.new(*args).each {|opt, arg| options[opt] = (arg || true)}
|
66
|
-
rescue StandardError => exc
|
67
|
-
raise WrongUsage.new nil, exc.message
|
68
|
-
end
|
69
|
-
|
70
|
-
# Let's just show the help to the user
|
71
|
-
raise WrongUsage.new(0, nil) if options['--help']
|
72
|
-
|
73
|
-
# Returning the options to the next level
|
74
|
-
options
|
75
|
-
end
|
76
|
-
|
77
|
-
def read_info_from_args(options)
|
78
|
-
# Parsing expre date
|
79
|
-
if options['--expires-in'] =~ /d|h|m|s/
|
80
|
-
|
81
|
-
val = 0
|
82
|
-
|
83
|
-
options['--expires-in'].scan(/(\d+\w)/) do |track|
|
84
|
-
_, num, what = /(\d+)(\w)/.match(track[0]).to_a
|
85
|
-
num = num.to_i
|
86
|
-
|
87
|
-
case what
|
88
|
-
when "d"; val += num * 86400
|
89
|
-
when "h"; val += num * 3600
|
90
|
-
when "m"; val += num * 60
|
91
|
-
when "s"; val += num
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
options['--expires-in'] = val
|
96
|
-
end
|
97
|
-
|
98
|
-
# Reading what to do from the user
|
99
|
-
command = ARGV.shift
|
100
|
-
raise WrongUsage.new(nil, "Need a command (eg.: `list', `listbuckets', etc)") if not command
|
101
|
-
|
102
|
-
key, file = ARGV
|
103
|
-
|
104
|
-
# Parsing the bucket name
|
105
|
-
bucket = nil
|
106
|
-
bucket, key = key.split(':') if key
|
107
|
-
|
108
|
-
# Returning things we need in the next level
|
109
|
-
[options, command, bucket, key, file]
|
110
|
-
end
|
111
|
-
end
|
112
|
-
end
|
data/lib/s3ranger/commands.rb
DELETED
@@ -1,114 +0,0 @@
|
|
1
|
-
require 's3ranger/exceptions'
|
2
|
-
require 's3ranger/sync'
|
3
|
-
require 'aws/s3'
|
4
|
-
|
5
|
-
|
6
|
-
module Commands
|
7
|
-
|
8
|
-
include S3Ranger
|
9
|
-
|
10
|
-
AVAILABLE_ACLS = [:public_read, :public_read_write, :private]
|
11
|
-
|
12
|
-
AVAILABLE_METHODS = ['read', 'get', 'put', 'write', 'delete']
|
13
|
-
|
14
|
-
def Commands._cmd_listbuckets args
|
15
|
-
args[:s3].buckets.each do |bkt|
|
16
|
-
puts "#{bkt.name}"
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def Commands._cmd_createbucket args
|
21
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
22
|
-
|
23
|
-
begin
|
24
|
-
params = {}
|
25
|
-
if acl = args[:options]['--acl']
|
26
|
-
raise WrongUsage.new(nil, "Invalid ACL. Should be any of #{EXISTING_ACLS.join ', '}") if not AVAILABLE_ACLS.include? acl
|
27
|
-
params.merge!({:acl => acl.to_sym})
|
28
|
-
end
|
29
|
-
|
30
|
-
args[:s3].buckets.create args[:bucket], params
|
31
|
-
rescue AWS::S3::Errors::BucketAlreadyExists => exc
|
32
|
-
raise FailureFeedback.new("Bucket `#{bucket}' already exists")
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def Commands._cmd_deletebucket args
|
37
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
38
|
-
|
39
|
-
# Getting the bucket
|
40
|
-
bucket_obj = args[:s3].buckets[args[:bucket]]
|
41
|
-
|
42
|
-
# Do not kill buckets with content unless explicitly asked
|
43
|
-
if not args[:options]['--force'] and bucket_obj.objects.count > 0
|
44
|
-
raise FailureFeedback.new("Cowardly refusing to remove non-empty bucket `#{bucket}'. Try with -f.")
|
45
|
-
end
|
46
|
-
|
47
|
-
begin
|
48
|
-
bucket_obj.delete!
|
49
|
-
rescue AWS::S3::Errors::AccessDenied => exc
|
50
|
-
raise FailureFeedback.new("Access Denied")
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def Commands._cmd_list args
|
55
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
56
|
-
args[:s3].buckets[args[:bucket]].objects.with_prefix(args[:key] || "").each do |object|
|
57
|
-
puts "#{object.key}\t#{object.content_length}\t#{object.last_modified}"
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
def Commands._cmd_delete args
|
62
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
63
|
-
raise WrongUsage.new(nil, "You need to inform a key") if not args[:key]
|
64
|
-
args[:s3].buckets[args[:bucket]].objects[args[:key]].delete
|
65
|
-
end
|
66
|
-
|
67
|
-
def Commands._cmd_url args
|
68
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
69
|
-
raise WrongUsage.new(nil, "You need to inform a key") if not args[:key]
|
70
|
-
|
71
|
-
method = args[:options]['--method'] || 'read'
|
72
|
-
raise WrongUsage.new(nil, "") unless AVAILABLE_METHODS.include? method
|
73
|
-
|
74
|
-
opts = {}
|
75
|
-
opts.merge!({:secure => args[:options]["--no-ssl"].nil?})
|
76
|
-
opts.merge!({:expires => args[:options]["--expires-in"]}) if args[:options]["--expires-in"]
|
77
|
-
p (args[:s3].buckets[args[:bucket]].objects[args[:key]].url_for method.to_sym, opts).to_s
|
78
|
-
end
|
79
|
-
|
80
|
-
def Commands._cmd_put args
|
81
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
82
|
-
raise WrongUsage.new(nil, "You need to inform a file") if not args[:file]
|
83
|
-
|
84
|
-
# key + file name
|
85
|
-
name = S3Ranger.safe_join [args[:key], File.basename(args[:file])]
|
86
|
-
args[:s3].buckets[args[:bucket]].objects[name].write Pathname.new(args[:file])
|
87
|
-
end
|
88
|
-
|
89
|
-
def Commands._cmd_get args
|
90
|
-
raise WrongUsage.new(nil, "You need to inform a bucket") if not args[:bucket]
|
91
|
-
raise WrongUsage.new(nil, "You need to inform a key") if not args[:key]
|
92
|
-
raise WrongUsage.new(nil, "You need to inform a file") if not args[:file]
|
93
|
-
|
94
|
-
# Saving the content to be downloaded to the current directory if the
|
95
|
-
# destination is a directory
|
96
|
-
path = File.absolute_path args[:file]
|
97
|
-
path = S3Ranger.safe_join [path, File.basename(args[:key])] if File.directory? path
|
98
|
-
|
99
|
-
File.open(path, 'wb') do |f|
|
100
|
-
begin
|
101
|
-
args[:s3].buckets[args[:bucket]].objects[args[:key]].read do |chunk| f.write(chunk) end
|
102
|
-
rescue AWS::S3::Errors::NoSuchBucket
|
103
|
-
raise FailureFeedback.new("There's no bucket named `#{args[:bucket]}'")
|
104
|
-
rescue AWS::S3::Errors::NoSuchKey
|
105
|
-
raise FailureFeedback.new("There's no key named `#{args[:key]}' in the bucket `#{args[:bucket]}'")
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
|
-
def Commands._cmd_sync args
|
111
|
-
cmd = SyncCommand.new args, *ARGV
|
112
|
-
cmd.run
|
113
|
-
end
|
114
|
-
end
|