s3sync 1.2.5 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/s3sync +67 -726
- data/lib/s3sync.rb +2 -0
- data/lib/s3sync/cli.rb +475 -0
- data/lib/s3sync/config.rb +98 -0
- data/lib/s3sync/exceptions.rb +55 -0
- data/lib/s3sync/sync.rb +371 -0
- data/lib/s3sync/util.rb +29 -0
- data/lib/s3sync/version.rb +27 -0
- metadata +177 -54
- data/CHANGELOG +0 -175
- data/README +0 -401
- data/README_s3cmd +0 -172
- data/Rakefile +0 -35
- data/bin/s3cmd +0 -245
- data/lib/HTTPStreaming.rb +0 -103
- data/lib/S3.rb +0 -707
- data/lib/S3_s3sync_mod.rb +0 -143
- data/lib/S3encoder.rb +0 -50
- data/lib/s3config.rb +0 -27
- data/lib/s3try.rb +0 -161
- data/lib/thread_generator.rb +0 -383
- data/lib/version.rb +0 -9
- data/setup.rb +0 -1585
data/lib/s3sync/util.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# s3sync - Tool belt for managing your S3 buckets
|
2
|
+
#
|
3
|
+
# The MIT License (MIT)
|
4
|
+
#
|
5
|
+
# Copyright (c) 2013 Lincoln de Sousa <lincoln@clarete.li>
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
8
|
+
# of this software and associated documentation files (the "Software"), to deal
|
9
|
+
# in the Software without restriction, including without limitation the rights
|
10
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
11
|
+
# copies of the Software, and to permit persons to whom the Software is
|
12
|
+
# furnished to do so, subject to the following conditions:
|
13
|
+
#
|
14
|
+
# The above copyright notice and this permission notice shall be included in
|
15
|
+
# all copies or substantial portions of the Software.
|
16
|
+
#
|
17
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
18
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
19
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
20
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
21
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
22
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
23
|
+
# THE SOFTWARE.
|
24
|
+
|
25
|
+
module S3Sync
|
26
|
+
def S3Sync.safe_join(parts)
|
27
|
+
File.join(*(parts.select {|v| !v.nil? && !v.empty? }))
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# s3sync - Tool belt for managing your S3 buckets
|
2
|
+
#
|
3
|
+
# The MIT License (MIT)
|
4
|
+
#
|
5
|
+
# Copyright (c) 2013 Lincoln de Sousa <lincoln@clarete.li>
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
8
|
+
# of this software and associated documentation files (the "Software"), to deal
|
9
|
+
# in the Software without restriction, including without limitation the rights
|
10
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
11
|
+
# copies of the Software, and to permit persons to whom the Software is
|
12
|
+
# furnished to do so, subject to the following conditions:
|
13
|
+
#
|
14
|
+
# The above copyright notice and this permission notice shall be included in
|
15
|
+
# all copies or substantial portions of the Software.
|
16
|
+
#
|
17
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
18
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
19
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
20
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
21
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
22
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
23
|
+
# THE SOFTWARE.
|
24
|
+
|
25
|
+
module S3Sync
|
26
|
+
VERSION = "2.0.0"
|
27
|
+
end
|
metadata
CHANGED
@@ -1,67 +1,190 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3sync
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 2.0.0
|
5
|
+
prerelease:
|
5
6
|
platform: ruby
|
6
|
-
authors:
|
7
|
-
-
|
7
|
+
authors:
|
8
|
+
- Lincoln de Sousa
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
12
|
+
date: 2013-09-25 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: aws-sdk
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: cmdparse
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: debugger
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: simplecov
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: rspec
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: bundler
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ~>
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '1.3'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ~>
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '1.3'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: rake
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: bump
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
description: Tool belt for managing your S3 buckets
|
143
|
+
email:
|
144
|
+
- lincoln@comum.org
|
145
|
+
executables:
|
19
146
|
- s3sync
|
20
|
-
- s3cmd
|
21
147
|
extensions: []
|
22
|
-
|
23
|
-
|
24
|
-
-
|
25
|
-
-
|
26
|
-
|
27
|
-
- lib/
|
28
|
-
- lib/
|
29
|
-
- lib/
|
30
|
-
- lib/
|
31
|
-
- lib/
|
32
|
-
|
33
|
-
|
34
|
-
-
|
35
|
-
- Rakefile
|
36
|
-
- setup.rb
|
37
|
-
- CHANGELOG
|
38
|
-
- README
|
39
|
-
- README_s3cmd
|
40
|
-
has_rdoc: true
|
41
|
-
homepage: http://s3sync.net/
|
148
|
+
extra_rdoc_files: []
|
149
|
+
files:
|
150
|
+
- bin/s3sync
|
151
|
+
- lib/s3sync.rb
|
152
|
+
- lib/s3sync/cli.rb
|
153
|
+
- lib/s3sync/config.rb
|
154
|
+
- lib/s3sync/exceptions.rb
|
155
|
+
- lib/s3sync/sync.rb
|
156
|
+
- lib/s3sync/util.rb
|
157
|
+
- lib/s3sync/version.rb
|
158
|
+
homepage: https://github.com/clarete/s3sync
|
159
|
+
licenses:
|
160
|
+
- MIT
|
42
161
|
post_install_message:
|
43
162
|
rdoc_options: []
|
44
|
-
|
45
|
-
require_paths:
|
163
|
+
require_paths:
|
46
164
|
- lib
|
47
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
165
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
166
|
+
none: false
|
167
|
+
requirements:
|
168
|
+
- - ! '>='
|
169
|
+
- !ruby/object:Gem::Version
|
170
|
+
version: '0'
|
171
|
+
segments:
|
172
|
+
- 0
|
173
|
+
hash: -1067117693450592295
|
174
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
175
|
+
none: false
|
176
|
+
requirements:
|
177
|
+
- - ! '>='
|
178
|
+
- !ruby/object:Gem::Version
|
179
|
+
version: '0'
|
180
|
+
segments:
|
181
|
+
- 0
|
182
|
+
hash: -1067117693450592295
|
59
183
|
requirements: []
|
60
|
-
|
61
|
-
|
62
|
-
rubygems_version: 1.0.1
|
184
|
+
rubyforge_project:
|
185
|
+
rubygems_version: 1.8.24
|
63
186
|
signing_key:
|
64
|
-
specification_version:
|
65
|
-
summary:
|
187
|
+
specification_version: 3
|
188
|
+
summary: s3sync is a library that aggregates a good range of features for managing
|
189
|
+
your Amazon S3 buckets. It also provides basic interactive client
|
66
190
|
test_files: []
|
67
|
-
|
data/CHANGELOG
DELETED
@@ -1,175 +0,0 @@
|
|
1
|
-
Change Log:
|
2
|
-
-----------
|
3
|
-
|
4
|
-
2006-09-29:
|
5
|
-
Added support for --expires and --cache-control. Eg:
|
6
|
-
--expires="Thu, 01 Dec 2007 16:00:00 GMT"
|
7
|
-
--cache-control="no-cache"
|
8
|
-
|
9
|
-
Thanks to Charles for pointing out the need for this, and supplying a patch
|
10
|
-
proving that it would be trivial to add =) Apologies for not including the short
|
11
|
-
form (-e) for the expires. I have a rule that options taking arguments should
|
12
|
-
use the long form.
|
13
|
-
----------
|
14
|
-
|
15
|
-
2006-10-04
|
16
|
-
Several minor debugs and edge cases.
|
17
|
-
Fixed a bug where retries didn't rewind the stream to start over.
|
18
|
-
----------
|
19
|
-
|
20
|
-
2006-10-12
|
21
|
-
Version 1.0.5
|
22
|
-
Finally figured out and fixed bug of trying to follow local symlink-to-directory.
|
23
|
-
Fixed a really nasty sorting discrepancy that caused problems when files started
|
24
|
-
with the same name as a directory.
|
25
|
-
Retry on connection-reset on the S3 side.
|
26
|
-
Skip files that we can't read instead of dying.
|
27
|
-
----------
|
28
|
-
|
29
|
-
2006-10-12
|
30
|
-
Version 1.0.6
|
31
|
-
Some GC voodoo to try and keep a handle on the memory footprint a little better.
|
32
|
-
There is still room for improvement here.
|
33
|
-
----------
|
34
|
-
|
35
|
-
2006-10-13
|
36
|
-
Version 1.0.7
|
37
|
-
Fixed symlink dirs being stored to S3 as real dirs (and failing with 400)
|
38
|
-
Added a retry catch for connection timeout error.
|
39
|
-
(Hopefully) caught a bug that expected every S3 listing to contain results
|
40
|
-
----------
|
41
|
-
|
42
|
-
2006-10-14
|
43
|
-
Version 1.0.8
|
44
|
-
Was testing for file? before symlink? in localnode.stream. This meant that for
|
45
|
-
symlink files it was trying to shove the real file contents into the symlink
|
46
|
-
body on s3.
|
47
|
-
----------
|
48
|
-
|
49
|
-
2006-10-14
|
50
|
-
Version 1.0.9
|
51
|
-
Woops, I was using "max-entries" for some reason but the proper header is
|
52
|
-
"max-keys". Not a big deal.
|
53
|
-
Broke out the S3try stuff into a separate file so I could re-use it for s3cmd.rb
|
54
|
-
----------
|
55
|
-
|
56
|
-
2006-10-16
|
57
|
-
Added a couple debug lines; not even enough to call it a version revision.
|
58
|
-
----------
|
59
|
-
|
60
|
-
2006-10-25
|
61
|
-
Version 1.0.10
|
62
|
-
UTF-8 fixes.
|
63
|
-
Catching a couple more retry-able errors in s3try (instead of aborting the
|
64
|
-
program).
|
65
|
-
----------
|
66
|
-
|
67
|
-
2006-10-26
|
68
|
-
Version 1.0.11
|
69
|
-
Revamped some details of the generators and comparator so that directories are
|
70
|
-
handled in a more exact and uniform fashion across local and S3.
|
71
|
-
----------
|
72
|
-
|
73
|
-
2006-11-28
|
74
|
-
Version 1.0.12
|
75
|
-
Added a couple more error catches to s3try.
|
76
|
-
----------
|
77
|
-
|
78
|
-
2007-01-08
|
79
|
-
Version 1.0.13
|
80
|
-
Numerous small changes to slash and path handling, in order to catch several
|
81
|
-
cases where "root" directory nodes were not being created on S3.
|
82
|
-
This makes restores work a lot more intuitively in many cases.
|
83
|
-
----------
|
84
|
-
|
85
|
-
2007-01-25
|
86
|
-
Version 1.0.14
|
87
|
-
Peter Fales' marker fix.
|
88
|
-
Also, markers should be decoded into native charset (because that's what s3
|
89
|
-
expects to see).
|
90
|
-
----------
|
91
|
-
|
92
|
-
2007-02-19
|
93
|
-
Version 1.1.0
|
94
|
-
*WARNING* Lots of path-handling changes. *PLEASE* test safely before you just
|
95
|
-
swap this in for your working 1.0.x version.
|
96
|
-
|
97
|
-
- Adding --exclude (and there was much rejoicing).
|
98
|
-
- Found Yet Another Leading Slash Bug with respect to local nodes. It was always
|
99
|
-
"recursing" into the first folder even if there was no trailing slash and -r
|
100
|
-
wasn't specified. What it should have done in this case is simply create a node
|
101
|
-
for the directory itself, then stop (not check the dir's contents).
|
102
|
-
- Local node canonicalization was (potentially) stripping the trailing slash,
|
103
|
-
which we need in order to make some decisios in the local generator.
|
104
|
-
- Fixed problem where it would prepend a "/" to s3 key names even with blank
|
105
|
-
prefix.
|
106
|
-
- Fixed S3->local when there's no "/" in the source so it doesn't try to create
|
107
|
-
a folder with the bucket name.
|
108
|
-
- Updated s3try and s3_s3sync_mod to allow SSL_CERT_FILE
|
109
|
-
----------
|
110
|
-
|
111
|
-
2007-02-22
|
112
|
-
Version 1.1.1
|
113
|
-
Fixed dumb regression bug caused by the S3->local bucket name fix in 1.1.0
|
114
|
-
----------
|
115
|
-
|
116
|
-
2007-02-25
|
117
|
-
Version 1.1.2
|
118
|
-
Added --progress
|
119
|
-
----------
|
120
|
-
|
121
|
-
2007-06-02
|
122
|
-
Version 1.1.3
|
123
|
-
IMPORTANT!
|
124
|
-
Pursuant to http://s3sync.net/forum/index.php?topic=49.0 , the tar.gz now
|
125
|
-
expands into its own sub-directory named "s3sync" instead of dumping all the
|
126
|
-
files into the current directory.
|
127
|
-
|
128
|
-
In the case of commands of the form:
|
129
|
-
s3sync -r somedir somebucket:
|
130
|
-
The root directory node in s3 was being stored as "somedir/" instead of "somedir"
|
131
|
-
which caused restores to mess up when you say:
|
132
|
-
s3sync -r somebucket: restoredir
|
133
|
-
The fix to this, by coincidence, actually makes s3fox work even *less* well with
|
134
|
-
s3sync. I really need to build my own xul+javascript s3 GUI some day.
|
135
|
-
|
136
|
-
Also fixed some of the NoMethodError stuff for when --progress is used
|
137
|
-
and caught Errno::ETIMEDOUT
|
138
|
-
----------
|
139
|
-
|
140
|
-
2007-07-12
|
141
|
-
Version 1.1.4
|
142
|
-
Added Alastair Brunton's yaml config code.
|
143
|
-
----------
|
144
|
-
|
145
|
-
2007-11-17
|
146
|
-
Version 1.2.1
|
147
|
-
Compatibility for S3 API revisions.
|
148
|
-
When retries are exhausted, emit an error.
|
149
|
-
Don't ever try to delete the 'root' local dir.
|
150
|
-
----------
|
151
|
-
|
152
|
-
2007-11-20
|
153
|
-
Version 1.2.2
|
154
|
-
Handle EU bucket 307 redirects (in s3try.rb)
|
155
|
-
--make-dirs added
|
156
|
-
----------
|
157
|
-
|
158
|
-
2007-11-20
|
159
|
-
Version 1.2.3
|
160
|
-
Fix SSL verification settings that broke in new S3 API.
|
161
|
-
----------
|
162
|
-
|
163
|
-
2008-01-06
|
164
|
-
Version 1.2.4
|
165
|
-
Run from any dir (search "here" for includes).
|
166
|
-
Search out s3config.yml in some likely places.
|
167
|
-
Reset connection (properly) on retry-able non-50x errors.
|
168
|
-
Fix calling format bug preventing it from working from yml.
|
169
|
-
Added http proxy support.
|
170
|
-
----------
|
171
|
-
|
172
|
-
2008-05-11
|
173
|
-
Version 1.2.5
|
174
|
-
Added option --no-md5
|
175
|
-
----------
|