s3ranger 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/s3ranger/cli.rb +18 -10
- data/lib/s3ranger/sync.rb +22 -34
- data/lib/s3ranger/util.rb +1 -1
- data/lib/s3ranger/version.rb +1 -1
- metadata +22 -24
- data/.gitignore +0 -17
- data/Gemfile +0 -4
- data/Gemfile.lock +0 -51
- data/LICENSE.txt +0 -22
- data/README.md +0 -29
- data/Rakefile +0 -1
- data/doc/old/README.txt +0 -406
- data/doc/old/README_s3cmd.txt +0 -172
- data/s3ranger.gemspec +0 -32
- data/s3ranger.yml.example +0 -2
- data/spec/fixtures/nodes/1.txt +0 -1
- data/spec/local_source_spec.rb +0 -55
- data/spec/main_spec.rb +0 -140
- data/spec/spec_helper.rb +0 -25
data/lib/s3ranger/cli.rb
CHANGED
@@ -62,6 +62,15 @@ module S3Ranger
|
|
62
62
|
|
63
63
|
u.join ''
|
64
64
|
end
|
65
|
+
|
66
|
+
protected
|
67
|
+
|
68
|
+
def parse_acl(opt)
|
69
|
+
@acl = nil
|
70
|
+
opt.on("-a", "--acl=ACL", "Options: #{AVAILABLE_ACLS.join ', '}") {|acl|
|
71
|
+
@acl = acl.to_sym
|
72
|
+
}
|
73
|
+
end
|
65
74
|
end
|
66
75
|
|
67
76
|
class ListBuckets < BaseCmd
|
@@ -86,12 +95,8 @@ module S3Ranger
|
|
86
95
|
|
87
96
|
@short_desc = "Create a new bucket under your user account"
|
88
97
|
|
89
|
-
@acl = nil
|
90
|
-
|
91
98
|
self.options = CmdParse::OptionParserWrapper.new do |opt|
|
92
|
-
opt
|
93
|
-
@acl = acl.to_sym
|
94
|
-
}
|
99
|
+
parse_acl(opt)
|
95
100
|
end
|
96
101
|
end
|
97
102
|
|
@@ -106,7 +111,7 @@ module S3Ranger
|
|
106
111
|
end
|
107
112
|
|
108
113
|
s3.buckets.create bucket, params
|
109
|
-
rescue AWS::S3::Errors::BucketAlreadyExists
|
114
|
+
rescue AWS::S3::Errors::BucketAlreadyExists
|
110
115
|
raise FailureFeedback.new("Bucket `#{bucket}' already exists")
|
111
116
|
end
|
112
117
|
end
|
@@ -175,7 +180,7 @@ module S3Ranger
|
|
175
180
|
collection = s3.buckets[bucket].objects.with_prefix(key || "")
|
176
181
|
|
177
182
|
if @max_entries > 0
|
178
|
-
collection = collection.page(:per_page =>
|
183
|
+
collection = collection.page(:per_page => @max_entries)
|
179
184
|
end
|
180
185
|
|
181
186
|
collection.each {|object|
|
@@ -236,7 +241,7 @@ module S3Ranger
|
|
236
241
|
|
237
242
|
opt.on("--expires-in=EXPR", "How long the link takes to expire. Format: <# of seconds> | [#d|#h|#m|#s]") { |expr|
|
238
243
|
val = 0
|
239
|
-
expr.scan
|
244
|
+
expr.scan(/(\d+\w)/) do |track|
|
240
245
|
_, num, what = /(\d+)(\w)/.match(track[0]).to_a
|
241
246
|
num = num.to_i
|
242
247
|
|
@@ -264,10 +269,10 @@ module S3Ranger
|
|
264
269
|
opts.merge!({:secure => @secure})
|
265
270
|
|
266
271
|
if @public
|
267
|
-
puts
|
272
|
+
puts s3.buckets[bucket].objects[key].public_url(opts).to_s
|
268
273
|
else
|
269
274
|
opts.merge!({:expires => @expires_in}) if @expires_in
|
270
|
-
puts
|
275
|
+
puts s3.buckets[bucket].objects[key].url_for(method.to_sym, opts).to_s
|
271
276
|
end
|
272
277
|
end
|
273
278
|
end
|
@@ -317,6 +322,7 @@ module S3Ranger
|
|
317
322
|
attr_accessor :keep
|
318
323
|
attr_accessor :dry_run
|
319
324
|
attr_accessor :verbose
|
325
|
+
attr_accessor :acl
|
320
326
|
|
321
327
|
def initialize
|
322
328
|
super 'sync', false, false
|
@@ -337,6 +343,8 @@ module S3Ranger
|
|
337
343
|
@keep = true
|
338
344
|
}
|
339
345
|
|
346
|
+
parse_acl(opt)
|
347
|
+
|
340
348
|
opt.on("-d", "--dry-run", "Do not download or exclude anything, just show what was planned. Implies `verbose`.") {
|
341
349
|
@dry_run = true
|
342
350
|
@verbose = true
|
data/lib/s3ranger/sync.rb
CHANGED
@@ -121,11 +121,11 @@ module S3Ranger
|
|
121
121
|
|
122
122
|
class SyncCommand
|
123
123
|
|
124
|
-
def
|
124
|
+
def self.cmp list1, list2
|
125
125
|
l1 = {}; list1.each {|e| l1[e.path] = e}
|
126
126
|
l2 = {}; list2.each {|e| l2[e.path] = e}
|
127
127
|
|
128
|
-
same, to_add_to_2
|
128
|
+
same, to_add_to_2 = [], []
|
129
129
|
|
130
130
|
l1.each do |key, value|
|
131
131
|
value2 = l2.delete key
|
@@ -151,7 +151,7 @@ module S3Ranger
|
|
151
151
|
|
152
152
|
def run
|
153
153
|
# Reading the source and destination using our helper method
|
154
|
-
if (source, destination, bucket =
|
154
|
+
if (source, destination, bucket = self.class.parse_params [@source, @destination]).nil?
|
155
155
|
raise WrongUsage.new(nil, 'Need a source and a destination')
|
156
156
|
end
|
157
157
|
|
@@ -159,7 +159,7 @@ module S3Ranger
|
|
159
159
|
source_tree, destination_tree = read_trees source, destination
|
160
160
|
|
161
161
|
# Getting the list of resources to be exchanged between the two peers
|
162
|
-
_, to_add, to_remove =
|
162
|
+
_, to_add, to_remove = self.class.cmp source_tree, destination_tree
|
163
163
|
|
164
164
|
# Removing the items matching the exclude pattern if requested
|
165
165
|
to_add.select! { |e|
|
@@ -180,35 +180,35 @@ module S3Ranger
|
|
180
180
|
end
|
181
181
|
end
|
182
182
|
|
183
|
-
def
|
183
|
+
def self.parse_params args
|
184
184
|
# Reading the arbitrary parameters from the command line and getting
|
185
185
|
# modifiable copies to parse
|
186
186
|
source, destination = args; return nil if source.nil? or destination.nil?
|
187
187
|
|
188
188
|
# Sync from one s3 to another is currently not supported
|
189
|
-
if
|
189
|
+
if remote_prefix? source and remote_prefix? destination
|
190
190
|
raise WrongUsage.new(nil, 'Both arguments can\'t be on S3')
|
191
191
|
end
|
192
192
|
|
193
193
|
# C'mon, there's rsync out there
|
194
|
-
if !
|
194
|
+
if !remote_prefix? source and !remote_prefix? destination
|
195
195
|
raise WrongUsage.new(nil, 'One argument must be on S3')
|
196
196
|
end
|
197
197
|
|
198
|
-
source, destination =
|
198
|
+
source, destination = process_destination source, destination
|
199
199
|
return [Location.new(*source), Location.new(*destination)]
|
200
200
|
end
|
201
201
|
|
202
|
-
def
|
202
|
+
def self.remote_prefix?(prefix)
|
203
203
|
# allow for dos-like things e.g. C:\ to be treated as local even with
|
204
204
|
# colon.
|
205
205
|
prefix.include? ':' and not prefix.match '^[A-Za-z]:[\\\\/]'
|
206
206
|
end
|
207
207
|
|
208
|
-
def
|
208
|
+
def self.process_file_destination source, destination, file=""
|
209
209
|
if not file.empty?
|
210
210
|
sub = (remote_prefix? source) ? source.split(":")[1] : source
|
211
|
-
file = file.gsub
|
211
|
+
file = file.gsub(/^#{sub}/, '')
|
212
212
|
end
|
213
213
|
|
214
214
|
# no slash on end of source means we need to append the last src dir to
|
@@ -234,7 +234,7 @@ module S3Ranger
|
|
234
234
|
end
|
235
235
|
end
|
236
236
|
|
237
|
-
def
|
237
|
+
def self.process_destination source, destination
|
238
238
|
source, destination = source.dup, destination.dup
|
239
239
|
|
240
240
|
# don't repeat slashes
|
@@ -242,21 +242,17 @@ module S3Ranger
|
|
242
242
|
destination.squeeze! '/'
|
243
243
|
|
244
244
|
# Making sure that local paths won't break our stuff later
|
245
|
-
source.gsub!
|
246
|
-
destination.gsub!
|
245
|
+
source.gsub!(/^\.\//, '')
|
246
|
+
destination.gsub!(/^\.\//, '')
|
247
247
|
|
248
248
|
# Parsing the final destination
|
249
|
-
destination =
|
249
|
+
destination = process_file_destination source, destination, ""
|
250
250
|
|
251
251
|
# here's where we find out what direction we're going
|
252
252
|
source_is_s3 = remote_prefix? source
|
253
253
|
|
254
|
-
# alias these variables to the other strings (in ruby = does not make
|
255
|
-
# copies of strings)
|
256
|
-
remote_prefix = source_is_s3 ? source : destination
|
257
|
-
local_prefix = source_is_s3 ? destination : source
|
258
|
-
|
259
254
|
# canonicalize the S3 stuff
|
255
|
+
remote_prefix = source_is_s3 ? source : destination
|
260
256
|
bucket, remote_prefix = remote_prefix.split ":"
|
261
257
|
remote_prefix ||= ""
|
262
258
|
|
@@ -269,19 +265,11 @@ module S3Ranger
|
|
269
265
|
end
|
270
266
|
|
271
267
|
def read_tree_remote location
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
}
|
278
|
-
rescue AWS::S3::Errors::NoSuchBucket
|
279
|
-
raise FailureFeedback.new("There's no bucket named `#{location.bucket}'")
|
280
|
-
rescue AWS::S3::Errors::NoSuchKey
|
281
|
-
raise FailureFeedback.new("There's no key named `#{location.path}' in the bucket `#{location.bucket}'")
|
282
|
-
rescue AWS::S3::Errors::AccessDenied
|
283
|
-
raise FailureFeedback.new("Access denied")
|
284
|
-
end
|
268
|
+
dir = location.path
|
269
|
+
dir += '/' if not dir.empty? or dir.end_with?('/')
|
270
|
+
@args.s3.buckets[location.bucket].objects.with_prefix(dir || "").to_a.collect {|obj|
|
271
|
+
Node.new location.path, obj.key, obj.content_length
|
272
|
+
}
|
285
273
|
end
|
286
274
|
|
287
275
|
def read_trees source, destination
|
@@ -304,7 +292,7 @@ module S3Ranger
|
|
304
292
|
|
305
293
|
unless @args.dry_run
|
306
294
|
if File.file? e.path
|
307
|
-
@args.s3.buckets[remote.bucket].objects[e.path].write Pathname.new
|
295
|
+
@args.s3.buckets[remote.bucket].objects[e.path].write Pathname.new(e.path), :acl => @args.acl
|
308
296
|
end
|
309
297
|
end
|
310
298
|
end
|
data/lib/s3ranger/util.rb
CHANGED
data/lib/s3ranger/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3ranger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-09-24 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk
|
@@ -123,6 +123,22 @@ dependencies:
|
|
123
123
|
- - ! '>='
|
124
124
|
- !ruby/object:Gem::Version
|
125
125
|
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: bump
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
126
142
|
description: Tool belt for managing your S3 buckets
|
127
143
|
email:
|
128
144
|
- lincoln@comum.org
|
@@ -131,15 +147,7 @@ executables:
|
|
131
147
|
extensions: []
|
132
148
|
extra_rdoc_files: []
|
133
149
|
files:
|
134
|
-
- .gitignore
|
135
|
-
- Gemfile
|
136
|
-
- Gemfile.lock
|
137
|
-
- LICENSE.txt
|
138
|
-
- README.md
|
139
|
-
- Rakefile
|
140
150
|
- bin/s3ranger
|
141
|
-
- doc/old/README.txt
|
142
|
-
- doc/old/README_s3cmd.txt
|
143
151
|
- lib/s3ranger.rb
|
144
152
|
- lib/s3ranger/cli.rb
|
145
153
|
- lib/s3ranger/config.rb
|
@@ -147,13 +155,7 @@ files:
|
|
147
155
|
- lib/s3ranger/sync.rb
|
148
156
|
- lib/s3ranger/util.rb
|
149
157
|
- lib/s3ranger/version.rb
|
150
|
-
|
151
|
-
- s3ranger.yml.example
|
152
|
-
- spec/fixtures/nodes/1.txt
|
153
|
-
- spec/local_source_spec.rb
|
154
|
-
- spec/main_spec.rb
|
155
|
-
- spec/spec_helper.rb
|
156
|
-
homepage: http://github.com/clarete/s3ranger
|
158
|
+
homepage: https://github.com/clarete/s3ranger
|
157
159
|
licenses:
|
158
160
|
- MIT
|
159
161
|
post_install_message:
|
@@ -168,7 +170,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
168
170
|
version: '0'
|
169
171
|
segments:
|
170
172
|
- 0
|
171
|
-
hash:
|
173
|
+
hash: -3161220676864238924
|
172
174
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
173
175
|
none: false
|
174
176
|
requirements:
|
@@ -177,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
177
179
|
version: '0'
|
178
180
|
segments:
|
179
181
|
- 0
|
180
|
-
hash:
|
182
|
+
hash: -3161220676864238924
|
181
183
|
requirements: []
|
182
184
|
rubyforge_project:
|
183
185
|
rubygems_version: 1.8.24
|
@@ -185,8 +187,4 @@ signing_key:
|
|
185
187
|
specification_version: 3
|
186
188
|
summary: s3ranger is a library that aggregates a good range of features for managing
|
187
189
|
your Amazon S3 buckets. It also provides basic interactive client
|
188
|
-
test_files:
|
189
|
-
- spec/fixtures/nodes/1.txt
|
190
|
-
- spec/local_source_spec.rb
|
191
|
-
- spec/main_spec.rb
|
192
|
-
- spec/spec_helper.rb
|
190
|
+
test_files: []
|
data/.gitignore
DELETED
data/Gemfile
DELETED
data/Gemfile.lock
DELETED
@@ -1,51 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
s3ranger (0.3.0)
|
5
|
-
aws-sdk
|
6
|
-
cmdparse
|
7
|
-
|
8
|
-
GEM
|
9
|
-
remote: https://rubygems.org/
|
10
|
-
specs:
|
11
|
-
aws-sdk (1.16.0)
|
12
|
-
json (~> 1.4)
|
13
|
-
nokogiri (< 1.6.0)
|
14
|
-
uuidtools (~> 2.1)
|
15
|
-
cmdparse (2.0.5)
|
16
|
-
columnize (0.3.6)
|
17
|
-
debugger (1.6.1)
|
18
|
-
columnize (>= 0.3.1)
|
19
|
-
debugger-linecache (~> 1.2.0)
|
20
|
-
debugger-ruby_core_source (~> 1.2.3)
|
21
|
-
debugger-linecache (1.2.0)
|
22
|
-
debugger-ruby_core_source (1.2.3)
|
23
|
-
diff-lcs (1.2.4)
|
24
|
-
json (1.8.0)
|
25
|
-
multi_json (1.7.9)
|
26
|
-
nokogiri (1.5.10)
|
27
|
-
rake (10.1.0)
|
28
|
-
rspec (2.14.1)
|
29
|
-
rspec-core (~> 2.14.0)
|
30
|
-
rspec-expectations (~> 2.14.0)
|
31
|
-
rspec-mocks (~> 2.14.0)
|
32
|
-
rspec-core (2.14.5)
|
33
|
-
rspec-expectations (2.14.2)
|
34
|
-
diff-lcs (>= 1.1.3, < 2.0)
|
35
|
-
rspec-mocks (2.14.3)
|
36
|
-
simplecov (0.7.1)
|
37
|
-
multi_json (~> 1.0)
|
38
|
-
simplecov-html (~> 0.7.1)
|
39
|
-
simplecov-html (0.7.1)
|
40
|
-
uuidtools (2.1.4)
|
41
|
-
|
42
|
-
PLATFORMS
|
43
|
-
ruby
|
44
|
-
|
45
|
-
DEPENDENCIES
|
46
|
-
bundler (~> 1.3)
|
47
|
-
debugger
|
48
|
-
rake
|
49
|
-
rspec
|
50
|
-
s3ranger!
|
51
|
-
simplecov
|
data/LICENSE.txt
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
Copyright (c) 2013 Lincoln de Sousa <lincoln@clarete.li>
|
2
|
-
|
3
|
-
MIT License
|
4
|
-
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
-
a copy of this software and associated documentation files (the
|
7
|
-
"Software"), to deal in the Software without restriction, including
|
8
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
-
permit persons to whom the Software is furnished to do so, subject to
|
11
|
-
the following conditions:
|
12
|
-
|
13
|
-
The above copyright notice and this permission notice shall be
|
14
|
-
included in all copies or substantial portions of the Software.
|
15
|
-
|
16
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
# S3Ranger
|
2
|
-
|
3
|
-
TODO: Write a gem description
|
4
|
-
|
5
|
-
## Installation
|
6
|
-
|
7
|
-
Add this line to your application's Gemfile:
|
8
|
-
|
9
|
-
gem 's3ranger'
|
10
|
-
|
11
|
-
And then execute:
|
12
|
-
|
13
|
-
$ bundle
|
14
|
-
|
15
|
-
Or install it yourself as:
|
16
|
-
|
17
|
-
$ gem install s3ranger
|
18
|
-
|
19
|
-
## Usage
|
20
|
-
|
21
|
-
TODO: Write usage instructions here
|
22
|
-
|
23
|
-
## Contributing
|
24
|
-
|
25
|
-
1. Fork it
|
26
|
-
2. Create your feature branch (`git checkout -b my-new-feature`)
|
27
|
-
3. Commit your changes (`git commit -am 'Add some feature'`)
|
28
|
-
4. Push to the branch (`git push origin my-new-feature`)
|
29
|
-
5. Create new Pull Request
|
data/Rakefile
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require "bundler/gem_tasks"
|
data/doc/old/README.txt
DELETED
@@ -1,406 +0,0 @@
|
|
1
|
-
Welcome to s3ranger.rb
|
2
|
-
--------------------
|
3
|
-
Home page, wiki, forum, bug reports, etc: http://s3sync.net
|
4
|
-
|
5
|
-
This is a ruby program that easily transfers directories between a local
|
6
|
-
directory and an S3 bucket:prefix. It behaves somewhat, but not precisely, like
|
7
|
-
the rsync program. In particular, it shares rsync's peculiar behavior that
|
8
|
-
trailing slashes on the source side are meaningful. See examples below.
|
9
|
-
|
10
|
-
One benefit over some other comparable tools is that s3ranger goes out of its way
|
11
|
-
to mirror the directory structure on S3. Meaning you don't *need* to use s3ranger
|
12
|
-
later in order to view your files on S3. You can just as easily use an S3
|
13
|
-
shell, a web browser (if you used the --public-read option), etc. Note that
|
14
|
-
s3ranger is NOT necessarily going to be able to read files you uploaded via some
|
15
|
-
other tool. This includes things uploaded with the old perl version! For best
|
16
|
-
results, start fresh!
|
17
|
-
|
18
|
-
s3ranger runs happily on linux, probably other *ix, and also Windows (except that
|
19
|
-
symlinks and permissions management features don't do anything on Windows). If
|
20
|
-
you get it running somewhere interesting let me know (see below)
|
21
|
-
|
22
|
-
s3ranger is free, and license terms are included in all the source files. If you
|
23
|
-
decide to make it better, or find bugs, please let me know.
|
24
|
-
|
25
|
-
The original inspiration for this tool is the perl script by the same name which
|
26
|
-
was made by Thorsten von Eicken (and later updated by me). This ruby program
|
27
|
-
does not share any components or logic from that utility; the only relation is
|
28
|
-
that it performs a similar task.
|
29
|
-
|
30
|
-
|
31
|
-
Examples:
|
32
|
-
---------
|
33
|
-
(using S3 bucket 'mybucket' and prefix 'pre')
|
34
|
-
Put the local etc directory itself into S3
|
35
|
-
s3ranger.rb -r /etc mybucket:pre
|
36
|
-
(This will yield S3 keys named pre/etc/...)
|
37
|
-
Put the contents of the local /etc dir into S3, rename dir:
|
38
|
-
s3ranger.rb -r /etc/ mybucket:pre/etcbackup
|
39
|
-
(This will yield S3 keys named pre/etcbackup/...)
|
40
|
-
Put contents of S3 "directory" etc into local dir
|
41
|
-
s3ranger.rb -r mybucket:pre/etc/ /root/etcrestore
|
42
|
-
(This will yield local files at /root/etcrestore/...)
|
43
|
-
Put the contents of S3 "directory" etc into a local dir named etc
|
44
|
-
s3ranger.rb -r mybucket:pre/etc /root
|
45
|
-
(This will yield local files at /root/etc/...)
|
46
|
-
Put S3 nodes under the key pre/etc/ to the local dir etcrestore
|
47
|
-
**and create local dirs even if S3 side lacks dir nodes**
|
48
|
-
s3ranger.rb -r --make-dirs mybucket:pre/etc/ /root/etcrestore
|
49
|
-
(This will yield local files at /root/etcrestore/...)
|
50
|
-
|
51
|
-
|
52
|
-
Prerequisites:
|
53
|
-
--------------
|
54
|
-
You need a functioning Ruby (>=1.8.4) installation, as well as the OpenSSL ruby
|
55
|
-
library (which may or may not come with your ruby).
|
56
|
-
|
57
|
-
How you get these items working on your system is really not any of my
|
58
|
-
business, but you might find the following things helpful. If you're using
|
59
|
-
Windows, the ruby site has a useful "one click installer" (although it takes
|
60
|
-
more clicks than that, really). On debian (and ubuntu, and other debian-like
|
61
|
-
things), there are apt packages available for ruby and the open ssl lib.
|
62
|
-
|
63
|
-
|
64
|
-
Your environment:
|
65
|
-
-----------------
|
66
|
-
s3ranger needs to know several interesting values to work right. It looks for
|
67
|
-
them in the following environment variables -or- a s3config.yml file.
|
68
|
-
In the yml case, the names need to be lowercase (see example file).
|
69
|
-
Furthermore, the yml is searched for in the following locations, in order:
|
70
|
-
$S3CONF/s3config.yml
|
71
|
-
$HOME/.s3conf/s3config.yml
|
72
|
-
/etc/s3conf/s3config.yml
|
73
|
-
|
74
|
-
Required:
|
75
|
-
AWS_ACCESS_KEY_ID
|
76
|
-
AWS_SECRET_ACCESS_KEY
|
77
|
-
|
78
|
-
If you don't know what these are, then s3ranger is probably not the
|
79
|
-
right tool for you to be starting out with.
|
80
|
-
Optional:
|
81
|
-
AWS_S3_HOST - I don't see why the default would ever be wrong
|
82
|
-
HTTP_PROXY_HOST,HTTP_PROXY_PORT,HTTP_PROXY_USER,HTTP_PROXY_PASSWORD - proxy
|
83
|
-
SSL_CERT_DIR - Where your Cert Authority keys live; for verification
|
84
|
-
SSL_CERT_FILE - If you have just one PEM file for CA verification
|
85
|
-
S3SYNC_RETRIES - How many HTTP errors to tolerate before exiting
|
86
|
-
S3SYNC_WAITONERROR - How many seconds to wait after an http error
|
87
|
-
S3SYNC_MIME_TYPES_FILE - Where is your mime.types file
|
88
|
-
S3SYNC_NATIVE_CHARSET - For example Windows-1252. Defaults to ISO-8859-1.
|
89
|
-
AWS_CALLING_FORMAT - Defaults to REGULAR
|
90
|
-
REGULAR # http://s3.amazonaws.com/bucket/key
|
91
|
-
SUBDOMAIN # http://bucket.s3.amazonaws.com/key
|
92
|
-
VANITY # http://<vanity_domain>/key
|
93
|
-
|
94
|
-
Important: For EU-located buckets you should set the calling format to SUBDOMAIN
|
95
|
-
Important: For US buckets with CAPS or other weird traits set the calling format
|
96
|
-
to REGULAR
|
97
|
-
|
98
|
-
I use "envdir" from the daemontools package to set up my env
|
99
|
-
variables easily: http://cr.yp.to/daemontools/envdir.html
|
100
|
-
For example:
|
101
|
-
envdir /root/s3ranger/env /root/s3ranger/s3ranger.rb -etc etc etc
|
102
|
-
I know there are other similar tools out there as well.
|
103
|
-
|
104
|
-
You can also just call it in a shell script where you have exported the vars
|
105
|
-
first such as:
|
106
|
-
#!/bin/bash
|
107
|
-
export AWS_ACCESS_KEY_ID=valueGoesHere
|
108
|
-
...
|
109
|
-
s3ranger.rb -etc etc etc
|
110
|
-
|
111
|
-
But by far the easiest (and newest) way to set this up is to put the name:value
|
112
|
-
pairs in a file named s3config.yml and let the yaml parser pick them up. There
|
113
|
-
is an .example file shipped with the tar.gz to show what a yaml file looks like.
|
114
|
-
Thanks to Alastair Brunton for this addition.
|
115
|
-
|
116
|
-
You can also use some combination of .yaml and environment variables, if you
|
117
|
-
want. Go nuts.
|
118
|
-
|
119
|
-
|
120
|
-
Management tasks
|
121
|
-
----------------
|
122
|
-
For low-level S3 operations not encapsulated by the sync paradigm, try the
|
123
|
-
companion utility s3cmd.rb. See README_s3cmd.txt.
|
124
|
-
|
125
|
-
|
126
|
-
About single files
|
127
|
-
------------------
|
128
|
-
s3ranger lacks the special case code that would be needed in order to handle a
|
129
|
-
source/dest that's a single file. This isn't one of the supported use cases so
|
130
|
-
don't expect it to work. You can use the companion utility s3cmd.rb for single
|
131
|
-
get/puts.
|
132
|
-
|
133
|
-
|
134
|
-
About Directories, the bane of any S3 sync-er
|
135
|
-
---------------------------------------------
|
136
|
-
In S3 there's no actual concept of folders, just keys and nodes. So, every tool
|
137
|
-
uses its own proprietary way of storing dir info (my scheme being the best
|
138
|
-
naturally) and in general the methods are not compatible.
|
139
|
-
|
140
|
-
If you populate S3 by some means *other than* s3ranger and then try to use s3ranger
|
141
|
-
to "get" the S3 stuff to a local filesystem, you will want to use the
|
142
|
-
--make-dirs option. This causes the local dirs to be created even if there is no
|
143
|
-
s3ranger-compatible directory node info stored on the S3 side. In other words,
|
144
|
-
local folders are conjured into existence whenever they are needed to make the
|
145
|
-
"get" succeed.
|
146
|
-
|
147
|
-
|
148
|
-
About MD5 hashes
|
149
|
-
----------------
|
150
|
-
s3ranger's normal operation is to compare the file size and MD5 hash of each item
|
151
|
-
to decide whether it needs syncing. On the S3 side, these hashes are stored and
|
152
|
-
returned to us as the "ETag" of each item when the bucket is listed, so it's
|
153
|
-
very easy. On the local side, the MD5 must be calculated by pushing every byte
|
154
|
-
in the file through the MD5 algorithm. This is CPU and IO intensive!
|
155
|
-
|
156
|
-
Thus you can specify the option --no-md5. This will compare the upload time on
|
157
|
-
S3 to the "last modified" time on the local item, and not do md5 calculations
|
158
|
-
locally at all. This might cause more transfers than are absolutely necessary.
|
159
|
-
For example if the file is "touched" to a newer modified date, but its contents
|
160
|
-
didn't change. Conversely if a file's contents are modified but the date is not
|
161
|
-
updated, then the sync will pass over it. Lastly, if your clock is very
|
162
|
-
different from the one on the S3 servers, then you may see unanticipated
|
163
|
-
behavior.
|
164
|
-
|
165
|
-
|
166
|
-
A word on SSL_CERT_DIR:
|
167
|
-
-----------------------
|
168
|
-
On my debian install I didn't find any root authority public keys. I installed
|
169
|
-
some by running this shell archive:
|
170
|
-
http://mirbsd.mirsolutions.de/cvs.cgi/src/etc/ssl.certs.shar
|
171
|
-
(You have to click download, and then run it wherever you want the certs to be
|
172
|
-
placed). I do not in any way assert that these certificates are good,
|
173
|
-
comprehensive, moral, noble, or otherwise correct. But I am using them.
|
174
|
-
|
175
|
-
If you don't set up a cert dir, and try to use ssl, then you'll 1) get an ugly
|
176
|
-
warning message slapped down by ruby, and 2) not have any protection AT ALL from
|
177
|
-
malicious servers posing as s3.amazonaws.com. Seriously... you want to get
|
178
|
-
this right if you're going to have any sensitive data being tossed around.
|
179
|
-
--
|
180
|
-
There is a debian package ca-certificates; this is what I'm using now.
|
181
|
-
apt-get install ca-certificates
|
182
|
-
and then use:
|
183
|
-
SSL_CERT_DIR=/etc/ssl/certs
|
184
|
-
|
185
|
-
You used to be able to use just one certificate, but recently AWS has started
|
186
|
-
using more than one CA.
|
187
|
-
|
188
|
-
|
189
|
-
Getting started:
|
190
|
-
----------------
|
191
|
-
Invoke by typing s3ranger.rb and you should get a nice usage screen.
|
192
|
-
Options can be specified in short or long form (except --delete, which has no
|
193
|
-
short form)
|
194
|
-
|
195
|
-
ALWAYS TEST NEW COMMANDS using --dryrun(-n) if you want to see what will be
|
196
|
-
affected before actually doing it. ESPECIALLY if you use --delete. Otherwise, do
|
197
|
-
not be surprised if you misplace a '/' or two and end up deleting all your
|
198
|
-
precious, precious files.
|
199
|
-
|
200
|
-
If you use the --public-read(-p) option, items sent to S3 will be ACL'd so that
|
201
|
-
anonymous web users can download them, given the correct URL. This could be
|
202
|
-
useful if you intend to publish directories of information for others to see.
|
203
|
-
For example, I use s3ranger to publish itself to its home on S3 via the following
|
204
|
-
command: s3ranger.rb -v -p publish/ ServEdge_pub:s3ranger Where the files live in a
|
205
|
-
local folder called "publish" and I wish them to be copied to the URL:
|
206
|
-
http://s3.amazonaws.com/ServEdge_pub/s3ranger/... If you use --ssl(-s) then your
|
207
|
-
connections with S3 will be encrypted. Otherwise your data will be sent in clear
|
208
|
-
form, i.e. easy to intercept by malicious parties.
|
209
|
-
|
210
|
-
If you want to prune items from the destination side which are not found on the
|
211
|
-
source side, you can use --delete. Always test this with -n first to make sure
|
212
|
-
the command line you specify is not going to do something terrible to your
|
213
|
-
cherished and irreplaceable data.
|
214
|
-
|
215
|
-
|
216
|
-
Updates and other discussion:
|
217
|
-
-----------------------------
|
218
|
-
The latest version of s3ranger should normally be at:
|
219
|
-
http://s3.amazonaws.com/ServEdge_pub/s3ranger/s3ranger.tar.gz
|
220
|
-
and the Amazon S3 forums probably have a few threads going on it at any given
|
221
|
-
time. I may not always see things posted to the threads, so if you want you can
|
222
|
-
contact me at gbs-s3@10forward.com too.
|
223
|
-
|
224
|
-
|
225
|
-
Change Log:
|
226
|
-
-----------
|
227
|
-
|
228
|
-
2006-09-29:
|
229
|
-
Added support for --expires and --cache-control. Eg:
|
230
|
-
--expires="Thu, 01 Dec 2007 16:00:00 GMT"
|
231
|
-
--cache-control="no-cache"
|
232
|
-
|
233
|
-
Thanks to Charles for pointing out the need for this, and supplying a patch
|
234
|
-
proving that it would be trivial to add =) Apologies for not including the short
|
235
|
-
form (-e) for the expires. I have a rule that options taking arguments should
|
236
|
-
use the long form.
|
237
|
-
----------
|
238
|
-
|
239
|
-
2006-10-04
|
240
|
-
Several minor debugs and edge cases.
|
241
|
-
Fixed a bug where retries didn't rewind the stream to start over.
|
242
|
-
----------
|
243
|
-
|
244
|
-
2006-10-12
|
245
|
-
Version 1.0.5
|
246
|
-
Finally figured out and fixed bug of trying to follow local symlink-to-directory.
|
247
|
-
Fixed a really nasty sorting discrepancy that caused problems when files started
|
248
|
-
with the same name as a directory.
|
249
|
-
Retry on connection-reset on the S3 side.
|
250
|
-
Skip files that we can't read instead of dying.
|
251
|
-
----------
|
252
|
-
|
253
|
-
2006-10-12
|
254
|
-
Version 1.0.6
|
255
|
-
Some GC voodoo to try and keep a handle on the memory footprint a little better.
|
256
|
-
There is still room for improvement here.
|
257
|
-
----------
|
258
|
-
|
259
|
-
2006-10-13
|
260
|
-
Version 1.0.7
|
261
|
-
Fixed symlink dirs being stored to S3 as real dirs (and failing with 400)
|
262
|
-
Added a retry catch for connection timeout error.
|
263
|
-
(Hopefully) caught a bug that expected every S3 listing to contain results
|
264
|
-
----------
|
265
|
-
|
266
|
-
2006-10-14
|
267
|
-
Version 1.0.8
|
268
|
-
Was testing for file? before symlink? in localnode.stream. This meant that for
|
269
|
-
symlink files it was trying to shove the real file contents into the symlink
|
270
|
-
body on s3.
|
271
|
-
----------
|
272
|
-
|
273
|
-
2006-10-14
|
274
|
-
Version 1.0.9
|
275
|
-
Woops, I was using "max-entries" for some reason but the proper header is
|
276
|
-
"max-keys". Not a big deal.
|
277
|
-
Broke out the S3try stuff into a separate file so I could re-use it for s3cmd.rb
|
278
|
-
----------
|
279
|
-
|
280
|
-
2006-10-16
|
281
|
-
Added a couple debug lines; not even enough to call it a version revision.
|
282
|
-
----------
|
283
|
-
|
284
|
-
2006-10-25
|
285
|
-
Version 1.0.10
|
286
|
-
UTF-8 fixes.
|
287
|
-
Catching a couple more retry-able errors in s3try (instead of aborting the
|
288
|
-
program).
|
289
|
-
----------
|
290
|
-
|
291
|
-
2006-10-26
|
292
|
-
Version 1.0.11
|
293
|
-
Revamped some details of the generators and comparator so that directories are
|
294
|
-
handled in a more exact and uniform fashion across local and S3.
|
295
|
-
----------
|
296
|
-
|
297
|
-
2006-11-28
|
298
|
-
Version 1.0.12
|
299
|
-
Added a couple more error catches to s3try.
|
300
|
-
----------
|
301
|
-
|
302
|
-
2007-01-08
|
303
|
-
Version 1.0.13
|
304
|
-
Numerous small changes to slash and path handling, in order to catch several
|
305
|
-
cases where "root" directory nodes were not being created on S3.
|
306
|
-
This makes restores work a lot more intuitively in many cases.
|
307
|
-
----------
|
308
|
-
|
309
|
-
2007-01-25
|
310
|
-
Version 1.0.14
|
311
|
-
Peter Fales' marker fix.
|
312
|
-
Also, markers should be decoded into native charset (because that's what s3
|
313
|
-
expects to see).
|
314
|
-
----------
|
315
|
-
|
316
|
-
2007-02-19
|
317
|
-
Version 1.1.0
|
318
|
-
*WARNING* Lots of path-handling changes. *PLEASE* test safely before you just
|
319
|
-
swap this in for your working 1.0.x version.
|
320
|
-
|
321
|
-
- Adding --exclude (and there was much rejoicing).
|
322
|
-
- Found Yet Another Leading Slash Bug with respect to local nodes. It was always
|
323
|
-
"recursing" into the first folder even if there was no trailing slash and -r
|
324
|
-
wasn't specified. What it should have done in this case is simply create a node
|
325
|
-
for the directory itself, then stop (not check the dir's contents).
|
326
|
-
- Local node canonicalization was (potentially) stripping the trailing slash,
|
327
|
-
which we need in order to make some decisios in the local generator.
|
328
|
-
- Fixed problem where it would prepend a "/" to s3 key names even with blank
|
329
|
-
prefix.
|
330
|
-
- Fixed S3->local when there's no "/" in the source so it doesn't try to create
|
331
|
-
a folder with the bucket name.
|
332
|
-
- Updated s3try and s3_s3ranger_mod to allow SSL_CERT_FILE
|
333
|
-
----------
|
334
|
-
|
335
|
-
2007-02-22
|
336
|
-
Version 1.1.1
|
337
|
-
Fixed dumb regression bug caused by the S3->local bucket name fix in 1.1.0
|
338
|
-
----------
|
339
|
-
|
340
|
-
2007-02-25
|
341
|
-
Version 1.1.2
|
342
|
-
Added --progress
|
343
|
-
----------
|
344
|
-
|
345
|
-
2007-06-02
|
346
|
-
Version 1.1.3
|
347
|
-
IMPORTANT!
|
348
|
-
Pursuant to http://s3sync.net/forum/index.php?topic=49.0 , the tar.gz now
|
349
|
-
expands into its own sub-directory named "s3ranger" instead of dumping all the
|
350
|
-
files into the current directory.
|
351
|
-
|
352
|
-
In the case of commands of the form:
|
353
|
-
s3ranger -r somedir somebucket:
|
354
|
-
The root directory node in s3 was being stored as "somedir/" instead of "somedir"
|
355
|
-
which caused restores to mess up when you say:
|
356
|
-
s3ranger -r somebucket: restoredir
|
357
|
-
The fix to this, by coincidence, actually makes s3fox work even *less* well with
|
358
|
-
s3ranger. I really need to build my own xul+javascript s3 GUI some day.
|
359
|
-
|
360
|
-
Also fixed some of the NoMethodError stuff for when --progress is used
|
361
|
-
and caught Errno::ETIMEDOUT
|
362
|
-
----------
|
363
|
-
|
364
|
-
2007-07-12
|
365
|
-
Version 1.1.4
|
366
|
-
Added Alastair Brunton's yaml config code.
|
367
|
-
----------
|
368
|
-
|
369
|
-
2007-11-17
|
370
|
-
Version 1.2.1
|
371
|
-
Compatibility for S3 API revisions.
|
372
|
-
When retries are exhausted, emit an error.
|
373
|
-
Don't ever try to delete the 'root' local dir.
|
374
|
-
----------
|
375
|
-
|
376
|
-
2007-11-20
|
377
|
-
Version 1.2.2
|
378
|
-
Handle EU bucket 307 redirects (in s3try.rb)
|
379
|
-
--make-dirs added
|
380
|
-
----------
|
381
|
-
|
382
|
-
2007-11-20
|
383
|
-
Version 1.2.3
|
384
|
-
Fix SSL verification settings that broke in new S3 API.
|
385
|
-
----------
|
386
|
-
|
387
|
-
2008-01-06
|
388
|
-
Version 1.2.4
|
389
|
-
Run from any dir (search "here" for includes).
|
390
|
-
Search out s3config.yml in some likely places.
|
391
|
-
Reset connection (properly) on retry-able non-50x errors.
|
392
|
-
Fix calling format bug preventing it from working from yml.
|
393
|
-
Added http proxy support.
|
394
|
-
----------
|
395
|
-
|
396
|
-
2008-05-11
|
397
|
-
Version 1.2.5
|
398
|
-
Added option --no-md5
|
399
|
-
----------
|
400
|
-
|
401
|
-
2008-06-16
|
402
|
-
Version 1.2.6
|
403
|
-
Catch connect errors and retry.
|
404
|
-
----------
|
405
|
-
|
406
|
-
FNORD
|
data/doc/old/README_s3cmd.txt
DELETED
@@ -1,172 +0,0 @@
|
|
1
|
-
Welcome to s3cmd.rb
|
2
|
-
-------------------
|
3
|
-
This is a ruby program that wraps S3 operations into a simple command-line tool.
|
4
|
-
It is inspired by things like rsh3ll, #sh3ll, etc., but shares no code from
|
5
|
-
them. It's meant as a companion utility to s3ranger.rb but could be used on its
|
6
|
-
own (provided you have read the other readme file and know how to use s3ranger in
|
7
|
-
theory).
|
8
|
-
|
9
|
-
I made this even though lots of other "shell"s exist, because I wanted a
|
10
|
-
single-operation utility, instead of a shell "environment". This lends itself
|
11
|
-
more to scripting, etc. Also the delete operation on rsh3ll seems to be borken
|
12
|
-
at the moment? =(
|
13
|
-
|
14
|
-
Users not yet familiar with s3ranger should read about that first, since s3cmd and
|
15
|
-
s3ranger share a tremendous amount of conventions and syntax. Particularly you
|
16
|
-
have to set up environment variables prior to calling s3cmd, and s3cmd also uses
|
17
|
-
the "bucket:key" syntax popularized by s3ranger. Many of the options are the same
|
18
|
-
too. Really, go read the other readme first if you haven't used s3ranger yet.
|
19
|
-
Otherwise you will become confused. It's OK, I'll wait.
|
20
|
-
|
21
|
-
....
|
22
|
-
|
23
|
-
In general, s3ranger and s3cmd complement each other. s3ranger is useful to perform
|
24
|
-
serious synchronization operations, and s3cmd allows you to do simple things
|
25
|
-
such as bucket management, listing, transferring single files, and the like.
|
26
|
-
|
27
|
-
Here is the usage, with examples to follow.
|
28
|
-
|
29
|
-
s3cmd.rb [options] <command> [arg(s)] version 1.0.0
|
30
|
-
--help -h --verbose -v --dryrun -n
|
31
|
-
--ssl -s --debug -d
|
32
|
-
|
33
|
-
Commands:
|
34
|
-
s3cmd.rb listbuckets [headers]
|
35
|
-
s3cmd.rb createbucket|deletebucket <bucket> [headers]
|
36
|
-
s3cmd.rb list <bucket>[:prefix] [max/page] [delimiter] [headers]
|
37
|
-
s3cmd.rb delete <bucket>:key [headers]
|
38
|
-
s3cmd.rb deleteall <bucket>[:prefix] [headers]
|
39
|
-
s3cmd.rb get|put <bucket>:key <file> [headers]
|
40
|
-
|
41
|
-
|
42
|
-
A note about [headers]
|
43
|
-
----------------------
|
44
|
-
For some S3 operations, such as "put", you might want to specify certain headers
|
45
|
-
to the request such as Cache-Control, Expires, x-amz-acl, etc. Rather than
|
46
|
-
supporting a load of separate command-line options for these, I just allow
|
47
|
-
header specification. So to upload a file with public-read access you could
|
48
|
-
say:
|
49
|
-
s3cmd.rb put MyBucket:TheFile.txt x-amz-acl:public-read
|
50
|
-
|
51
|
-
If you don't need to add any particular headers then you can just ignore this
|
52
|
-
whole [headers] thing and pretend it's not there. This is somewhat of an
|
53
|
-
advanced option.
|
54
|
-
|
55
|
-
|
56
|
-
Examples
|
57
|
-
--------
|
58
|
-
List all the buckets your account owns:
|
59
|
-
s3cmd.rb listbuckets
|
60
|
-
|
61
|
-
Create a new bucket:
|
62
|
-
s3cmd.rb createbucket BucketName
|
63
|
-
|
64
|
-
Create a new bucket in the EU:
|
65
|
-
s3cmd.rb createbucket BucketName EU
|
66
|
-
|
67
|
-
Find out the location constraint of a bucket:
|
68
|
-
s3cmd.rb location BucketName
|
69
|
-
|
70
|
-
Delete an old bucket you don't want any more:
|
71
|
-
s3cmd.rb deletebucket BucketName
|
72
|
-
|
73
|
-
Find out what's in a bucket, 10 lines at a time:
|
74
|
-
s3cmd.rb list BucketName 10
|
75
|
-
|
76
|
-
Only look in a particular prefix:
|
77
|
-
s3cmd.rb list BucketName:startsWithThis
|
78
|
-
|
79
|
-
Look in the virtual "directory" named foo;
|
80
|
-
lists sub-"directories" and keys that are at this level.
|
81
|
-
Note that if you specify a delimiter you must specify a max before it.
|
82
|
-
(until I make the options parsing smarter)
|
83
|
-
s3cmd.rb list BucketName:foo/ 10 /
|
84
|
-
|
85
|
-
Delete a key:
|
86
|
-
s3cmd.rb delete BucketName:AKey
|
87
|
-
|
88
|
-
Delete all keys that match (like a combo between list and delete):
|
89
|
-
s3cmd.rb deleteall BucketName:SomePrefix
|
90
|
-
|
91
|
-
Only pretend you're going to delete all keys that match, but list them:
|
92
|
-
s3cmd.rb --dryrun deleteall BucketName:SomePrefix
|
93
|
-
|
94
|
-
Delete all keys in a bucket (leaving the bucket):
|
95
|
-
s3cmd.rb deleteall BucketName
|
96
|
-
|
97
|
-
Get a file from S3 and store it to a local file
|
98
|
-
s3cmd.rb get BucketName:TheFileOnS3.txt ALocalFile.txt
|
99
|
-
|
100
|
-
Put a local file up to S3
|
101
|
-
Note we don't automatically set mime type, etc.
|
102
|
-
NOTE that the order of the options doesn't change. S3 stays first!
|
103
|
-
s3cmd.rb put BucketName:TheFileOnS3.txt ALocalFile.txt
|
104
|
-
|
105
|
-
|
106
|
-
Change Log:
|
107
|
-
-----------
|
108
|
-
2006-10-14:
|
109
|
-
Created.
|
110
|
-
-----------
|
111
|
-
|
112
|
-
2006-10-16
|
113
|
-
Version 1.0.1
|
114
|
-
Force content length to a string value since some ruby's don't convert it right.
|
115
|
-
-----------
|
116
|
-
|
117
|
-
2006-10-25
|
118
|
-
UTF-8 fixes.
|
119
|
-
-----------
|
120
|
-
|
121
|
-
2006-11-28
|
122
|
-
Version 1.0.3
|
123
|
-
Added a couple more error catches to s3try.
|
124
|
-
----------
|
125
|
-
|
126
|
-
2007-01-25
|
127
|
-
Version 1.0.4
|
128
|
-
Peter Fales' marker fix.
|
129
|
-
Also, markers should be decoded into native charset (because that's what s3
|
130
|
-
expects to see).
|
131
|
-
----------
|
132
|
-
|
133
|
-
2007-02-19
|
134
|
-
- Updated s3try and s3_s3ranger_mod to allow SSL_CERT_FILE
|
135
|
-
----------
|
136
|
-
|
137
|
-
2007-2-25
|
138
|
-
Added --progress
|
139
|
-
----------
|
140
|
-
|
141
|
-
2007-07-12
|
142
|
-
Version 1.0.6
|
143
|
-
Added Alastair Brunton's yaml config code.
|
144
|
-
----------
|
145
|
-
|
146
|
-
2007-11-17
|
147
|
-
Version 1.2.1
|
148
|
-
Compatibility for S3 API revisions.
|
149
|
-
When retries are exhausted, emit an error.
|
150
|
-
----------
|
151
|
-
|
152
|
-
2007-11-20
|
153
|
-
Version 1.2.2
|
154
|
-
Handle EU bucket 307 redirects (in s3try.rb)
|
155
|
-
----------
|
156
|
-
|
157
|
-
2007-11-20
|
158
|
-
Version 1.2.3
|
159
|
-
Fix SSL verification settings that broke in new S3 API.
|
160
|
-
----------
|
161
|
-
|
162
|
-
2008-01-06
|
163
|
-
Version 1.2.4
|
164
|
-
Run from any dir (search "here" for includes).
|
165
|
-
Search out s3config.yml in some likely places.
|
166
|
-
Reset connection (properly) on retry-able non-50x errors.
|
167
|
-
Fix calling format bug preventing it from working from yml.
|
168
|
-
Added http proxy support.
|
169
|
-
----------
|
170
|
-
|
171
|
-
|
172
|
-
FNORD
|
data/s3ranger.gemspec
DELETED
@@ -1,32 +0,0 @@
|
|
1
|
-
# -*- mode: ruby; coding: utf-8; -*-
|
2
|
-
lib = File.expand_path('../lib', __FILE__)
|
3
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require 's3ranger/version'
|
5
|
-
|
6
|
-
Gem::Specification.new do |spec|
|
7
|
-
spec.name = "s3ranger"
|
8
|
-
spec.version = S3Ranger::VERSION
|
9
|
-
spec.authors = ["Lincoln de Sousa"]
|
10
|
-
spec.email = ["lincoln@comum.org"]
|
11
|
-
spec.description = 'Tool belt for managing your S3 buckets'
|
12
|
-
spec.summary = 's3ranger is a library that aggregates a good range of features for managing your Amazon S3 buckets. It also provides basic interactive client'
|
13
|
-
|
14
|
-
spec.homepage = "http://github.com/clarete/s3ranger"
|
15
|
-
spec.license = "MIT"
|
16
|
-
|
17
|
-
spec.files = `git ls-files`.split($/)
|
18
|
-
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
|
-
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
20
|
-
spec.require_paths = ["lib"]
|
21
|
-
|
22
|
-
# Library requirements
|
23
|
-
spec.add_dependency "aws-sdk"
|
24
|
-
spec.add_dependency "cmdparse"
|
25
|
-
|
26
|
-
# Development requirements
|
27
|
-
spec.add_development_dependency "debugger"
|
28
|
-
spec.add_development_dependency "simplecov"
|
29
|
-
spec.add_development_dependency "rspec"
|
30
|
-
spec.add_development_dependency "bundler", "~> 1.3"
|
31
|
-
spec.add_development_dependency "rake"
|
32
|
-
end
|
data/s3ranger.yml.example
DELETED
data/spec/fixtures/nodes/1.txt
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
s3ranger
|
data/spec/local_source_spec.rb
DELETED
@@ -1,55 +0,0 @@
|
|
1
|
-
require 'spec_helper.rb'
|
2
|
-
require 's3ranger/sync'
|
3
|
-
|
4
|
-
include S3Ranger
|
5
|
-
|
6
|
-
|
7
|
-
describe "Local file system IO" do
|
8
|
-
|
9
|
-
it "should list local files" do
|
10
|
-
|
11
|
-
# Given that I have remote source and a local destination with a couple
|
12
|
-
# files
|
13
|
-
source = "mybucket:path"
|
14
|
-
destination = directory "directory2"
|
15
|
-
file destination, "file1.txt", "First file"
|
16
|
-
file destination, "file2.txt", "Second file"
|
17
|
-
|
18
|
-
# When I create a new local directory based on that path
|
19
|
-
local = LocalDirectory.new destination
|
20
|
-
|
21
|
-
# Then I see that the directory nodes contain both their parent paths and
|
22
|
-
# their names
|
23
|
-
local.list_files.should be_eql [
|
24
|
-
Node.new(fixture("directory2"), "file1.txt", 10),
|
25
|
-
Node.new(fixture("directory2"), "file2.txt", 11),
|
26
|
-
]
|
27
|
-
|
28
|
-
rm destination
|
29
|
-
end
|
30
|
-
|
31
|
-
it "should skip local folders while listing files" do
|
32
|
-
# Given that I have remote source and a local destination with files
|
33
|
-
source = "mybucket:path"
|
34
|
-
destination = directory "directory2"
|
35
|
-
file destination, "file1.txt", "First file"
|
36
|
-
file destination, "file2.txt", "Second file"
|
37
|
-
|
38
|
-
# And with a sub-directory
|
39
|
-
subdir = directory "directory2/subd"
|
40
|
-
file subdir, "sub1.txt", "Sub content"
|
41
|
-
|
42
|
-
# When I create a new local directory based on that path
|
43
|
-
local = LocalDirectory.new destination
|
44
|
-
|
45
|
-
# Then I see that the directory nodes contain both their parent paths and
|
46
|
-
# their names
|
47
|
-
local.list_files.should be_eql [
|
48
|
-
Node.new(fixture("directory2"), "file1.txt", 10),
|
49
|
-
Node.new(fixture("directory2"), "file2.txt", 11),
|
50
|
-
Node.new(fixture("directory2"), "subd/sub1.txt", 11),
|
51
|
-
]
|
52
|
-
|
53
|
-
rm destination
|
54
|
-
end
|
55
|
-
end
|
data/spec/main_spec.rb
DELETED
@@ -1,140 +0,0 @@
|
|
1
|
-
require 'spec_helper.rb'
|
2
|
-
require 's3ranger/cli'
|
3
|
-
require 's3ranger/config'
|
4
|
-
require 's3ranger/sync'
|
5
|
-
|
6
|
-
include S3Ranger
|
7
|
-
|
8
|
-
describe "Parsing command line arguments" do
|
9
|
-
|
10
|
-
describe "Processing the final destination based on how the user expressed the source" do
|
11
|
-
|
12
|
-
it "Put the local etc directory itself into S3" do
|
13
|
-
source = "/etc"
|
14
|
-
destination = "mybucket:pre"
|
15
|
-
|
16
|
-
# This will yield S3 keys named pre/etc/...
|
17
|
-
SyncCommand.process_destination(source, destination).should be_eql ["/etc", ["pre/etc/", "mybucket"]]
|
18
|
-
end
|
19
|
-
|
20
|
-
it "Put the contents of the local /etc dir into S3, rename dir" do
|
21
|
-
source = "/etc/"
|
22
|
-
destination = "mybucket:pre/etcbackup"
|
23
|
-
|
24
|
-
# This will yield S3 keys named pre/etcbackup/...
|
25
|
-
SyncCommand.process_destination(source, destination).should be_eql ["/etc/", ["pre/etcbackup/", "mybucket"]]
|
26
|
-
end
|
27
|
-
|
28
|
-
it "Put contents of S3 \"directory\" etc into local dir" do
|
29
|
-
source = "mybucket:pre/etc/"
|
30
|
-
destination = "/root/etcrestore"
|
31
|
-
|
32
|
-
# This will yield local files at /root/etcrestore/...
|
33
|
-
SyncCommand.process_destination(source, destination).should be_eql [["pre/etc/", "mybucket"], "/root/etcrestore/"]
|
34
|
-
end
|
35
|
-
|
36
|
-
it "Put the contents of S3 \"directory\" etc into a local dir named etc" do
|
37
|
-
source = "mybucket:pre/etc"
|
38
|
-
destination = "/root"
|
39
|
-
|
40
|
-
# This will yield local files at /root/etc/...
|
41
|
-
SyncCommand.process_destination(source, destination).should be_eql [["pre/etc", "mybucket"], "/root/etc/"]
|
42
|
-
end
|
43
|
-
|
44
|
-
it "Put S3 nodes under the key pre/etc/ to the local dir etcrestore" do
|
45
|
-
source = "mybucket:pre/etc/"
|
46
|
-
destination = "/root/etcrestore"
|
47
|
-
|
48
|
-
# This will yield local files at /root/etcrestore/...
|
49
|
-
SyncCommand.process_destination(source, destination).should be_eql [["pre/etc/", "mybucket"], "/root/etcrestore/"]
|
50
|
-
end
|
51
|
-
|
52
|
-
it "Put S3 nodes under an empty key (root) to the local dir /tmp/lib" do
|
53
|
-
source = "mybucket:"
|
54
|
-
destination = "/tmp/lib"
|
55
|
-
|
56
|
-
# This will yield local files at /root/etcrestore/...
|
57
|
-
SyncCommand.process_destination(source, destination).should be_eql [["", "mybucket"], "/tmp/lib/"]
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
it "Should calculate the right destination for each path" do
|
62
|
-
file = "pre/etc/sub/path/blah.txt" # This is how it comes from s3
|
63
|
-
source = "mybucket:pre/etc/"
|
64
|
-
destination = "/root/etcrestore"
|
65
|
-
|
66
|
-
SyncCommand.process_file_destination(source, destination, file).should be_eql "/root/etcrestore/sub/path/blah.txt"
|
67
|
-
end
|
68
|
-
|
69
|
-
it "Put S3 files under an empty key (root) to the local dir /tmp/lib" do
|
70
|
-
source = "mybucket:"
|
71
|
-
destination = "/tmp/lib"
|
72
|
-
file = "myfile.rb"
|
73
|
-
|
74
|
-
# This will yield local files at /tmp/lib/...
|
75
|
-
SyncCommand.process_file_destination(source, destination, file).should be_eql "/tmp/lib/myfile.rb"
|
76
|
-
end
|
77
|
-
|
78
|
-
it "Returning locations based on the parsed destination" do
|
79
|
-
source = "/etc"
|
80
|
-
destination = "mybucket:pre"
|
81
|
-
|
82
|
-
# When I parse the above arguments using the SyncCommand
|
83
|
-
src_location, dst_location = SyncCommand.parse_params [source, destination]
|
84
|
-
|
85
|
-
# Then I see I got the locations with the right params
|
86
|
-
src_location.should be_eql S3Ranger::Location.new("/etc")
|
87
|
-
dst_location.should be_eql S3Ranger::Location.new("pre/etc/", "mybucket")
|
88
|
-
end
|
89
|
-
|
90
|
-
it "Location should be parsed when it is remote with no path" do
|
91
|
-
source = "/etc"
|
92
|
-
destination = "mybucket:"
|
93
|
-
|
94
|
-
# When I parse the above arguments using the SyncCommand
|
95
|
-
src_location, dst_location = SyncCommand.parse_params [source, destination]
|
96
|
-
|
97
|
-
# Then I see I got the locations with the right params
|
98
|
-
src_location.should be_eql S3Ranger::Location.new("/etc")
|
99
|
-
dst_location.should be_eql S3Ranger::Location.new("etc/", "mybucket")
|
100
|
-
end
|
101
|
-
|
102
|
-
it "should be possible to detect if a location is remote" do
|
103
|
-
SyncCommand.remote_prefix?("bucket:prefix").should be_true
|
104
|
-
SyncCommand.remote_prefix?("path").should be_false
|
105
|
-
SyncCommand.remote_prefix?("C://blah").should be_false # We support windows, LOL
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
describe "Comparing file lists" do
|
110
|
-
|
111
|
-
it "should be possible to describe nodes with their paths and size" do
|
112
|
-
|
113
|
-
# Full test
|
114
|
-
node = Node.new "path//to", "file1", 10
|
115
|
-
node.path.should be_eql "file1"
|
116
|
-
node.full.should be_eql "path/to/file1"
|
117
|
-
node.size.should be_eql 10
|
118
|
-
|
119
|
-
# Alternative constructor scenarios
|
120
|
-
node = Node.new "", "file1", 10
|
121
|
-
node.path.should be_eql "file1"
|
122
|
-
end
|
123
|
-
|
124
|
-
it "should be possible to compare two lists of files" do
|
125
|
-
|
126
|
-
# Given that I have two lists of Nodes to compare
|
127
|
-
list1 = [Node.new("", "file1", 10), Node.new("", "file2", 12), Node.new("", "file3", 12)]
|
128
|
-
list2 = [Node.new("", "file1", 10), Node.new("", "file2", 22), Node.new("", "file4", 22),]
|
129
|
-
|
130
|
-
# When I compare those two file lists
|
131
|
-
same_in_both, to_be_added_to_list2, to_be_removed_from_list2 = SyncCommand.cmp list1, list2
|
132
|
-
|
133
|
-
# Then I see that the three lists that I requested were returned with the
|
134
|
-
# right content
|
135
|
-
same_in_both.should == [Node.new("", "file1", 10)] # Just testing our == operator
|
136
|
-
same_in_both.should be_eql [Node.new("", "file1", 10)]
|
137
|
-
to_be_added_to_list2.should be_eql [Node.new("", "file2", 12), Node.new("", "file3", 12)]
|
138
|
-
to_be_removed_from_list2.should be_eql [Node.new("", "file4", 22)]
|
139
|
-
end
|
140
|
-
end
|
data/spec/spec_helper.rb
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
require 'fileutils'
|
2
|
-
require 'simplecov'
|
3
|
-
SimpleCov.start
|
4
|
-
|
5
|
-
def fixture *args
|
6
|
-
File.join File.dirname(__FILE__), "fixtures", *args
|
7
|
-
end
|
8
|
-
|
9
|
-
def directory path
|
10
|
-
full = fixture(path)
|
11
|
-
FileUtils::mkdir_p full
|
12
|
-
return full
|
13
|
-
end
|
14
|
-
|
15
|
-
def file *args
|
16
|
-
file = File.join *args[0..-2]
|
17
|
-
directory File.dirname(file)
|
18
|
-
File.open(file, 'w') {|f| f.write args[-1] }
|
19
|
-
return file
|
20
|
-
end
|
21
|
-
|
22
|
-
|
23
|
-
def rm path
|
24
|
-
FileUtils::rm_rf path
|
25
|
-
end
|