s3_meta_sync 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/lib/s3_meta_sync/version.rb +1 -1
- data/lib/s3_meta_sync.rb +34 -18
- data.tar.gz.sig +2 -2
- metadata +2 -2
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b2ae988de36b8fabaa5dfa6f19726fc971b46133
|
4
|
+
data.tar.gz: 50e7f0a149cb5ed3bd3d6c09c9b32e2998118996
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fa22acee1c5ecac7af983b52fa856948fb7c98f8b66bd374f5f869b24d08d725f6c083ebd89c6c440df3cd14ec50cfa0603546c8bfe899cf3bf1e5ddd09428fb
|
7
|
+
data.tar.gz: c6e7d3927654ec728757431865ced2cd2a08db8f5b78e9fe9a306f55a9b4c3527af8d114dffd5c2d3aa1fb92d5efdef22682787597bbf3a62270dcc71d3b42de
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/lib/s3_meta_sync/version.rb
CHANGED
data/lib/s3_meta_sync.rb
CHANGED
@@ -34,31 +34,33 @@ module S3MetaSync
|
|
34
34
|
remote_info = begin
|
35
35
|
download_meta(destination)
|
36
36
|
rescue RemoteWithoutMeta
|
37
|
+
log "Remote has no .s3-meta-sync, uploading everything"
|
37
38
|
{}
|
38
39
|
end
|
39
40
|
generate_meta(source)
|
40
41
|
local_info = read_meta(source)
|
42
|
+
upload = local_info.select { |path, md5| remote_info[path] != md5 }.map(&:first)
|
43
|
+
delete = remote_info.keys - local_info.keys
|
41
44
|
|
42
|
-
|
43
|
-
|
44
|
-
upload_file(source, path, destination)
|
45
|
-
end
|
46
|
-
|
47
|
-
delete_remote_files(destination, remote_info.keys - local_info.keys)
|
48
|
-
|
45
|
+
upload_files(source, destination, upload)
|
46
|
+
delete_remote_files(destination, delete)
|
49
47
|
upload_file(source, META_FILE, destination)
|
50
48
|
end
|
51
49
|
|
52
50
|
def upload_file(source, path, destination)
|
51
|
+
log "Uploading #{path}"
|
53
52
|
s3.objects["#{destination}/#{path}"].write File.read("#{source}/#{path}"), :acl => :public_read
|
54
53
|
end
|
55
54
|
|
56
55
|
def delete_remote_files(remote, paths)
|
56
|
+
paths.each { |path| log "Deleting #{@bucket}:#{remote}/#{path}" }
|
57
57
|
s3.objects.delete paths.map { |path| "#{remote}/#{path}" }
|
58
58
|
end
|
59
59
|
|
60
60
|
def delete_local_files(local, paths)
|
61
|
-
|
61
|
+
paths = paths.map { |path| "#{local}/#{path}" }
|
62
|
+
paths.each { |path| log "Deleting #{path}" }
|
63
|
+
File.delete(*paths)
|
62
64
|
end
|
63
65
|
|
64
66
|
def s3
|
@@ -94,6 +96,7 @@ module S3MetaSync
|
|
94
96
|
end
|
95
97
|
|
96
98
|
def download_content(path)
|
99
|
+
log "Downloading #{path}"
|
97
100
|
url = "https://s3#{"-#{region}" if region}.amazonaws.com/#{@bucket}/#{path}"
|
98
101
|
open(url).read
|
99
102
|
rescue OpenURI::HTTPError
|
@@ -103,27 +106,39 @@ module S3MetaSync
|
|
103
106
|
def download(source, destination)
|
104
107
|
remote_info = download_meta(source)
|
105
108
|
generate_meta(destination)
|
106
|
-
local_info = read_meta(destination)
|
107
|
-
|
108
|
-
|
109
|
-
next if local_info[path] == md5
|
110
|
-
download_file(source, path, destination)
|
111
|
-
end
|
112
|
-
|
113
|
-
delete_local_files(destination, local_info.keys - remote_info.keys)
|
109
|
+
local_info = read_meta(destination)
|
110
|
+
download = remote_info.select { |path, md5| local_info[path] != md5 }.map(&:first)
|
111
|
+
delete = local_info.keys - remote_info.keys
|
114
112
|
|
113
|
+
download_files(source, destination, download)
|
114
|
+
delete_local_files(destination, delete)
|
115
115
|
download_file(source, META_FILE, destination)
|
116
|
+
delete_empty_folders(destination)
|
117
|
+
end
|
116
118
|
|
119
|
+
def delete_empty_folders(destination)
|
117
120
|
`find #{destination} -depth -empty -delete`
|
118
121
|
end
|
119
122
|
|
123
|
+
def download_files(source, destination, paths)
|
124
|
+
in_parallel(:threads, paths) { |path| download_file(source, path, destination) }
|
125
|
+
end
|
126
|
+
|
127
|
+
def upload_files(source, destination, paths)
|
128
|
+
in_parallel(:processes, paths) { |path| upload_file(source, path, destination) }
|
129
|
+
end
|
130
|
+
|
120
131
|
def region
|
121
132
|
@config[:region] unless @config[:region].to_s.empty?
|
122
133
|
end
|
123
134
|
|
124
|
-
def
|
135
|
+
def in_parallel(way, data, &block)
|
125
136
|
processes = [@config[:parallel] || 10, data.size].min
|
126
|
-
Parallel.each(data, :
|
137
|
+
Parallel.each(data, :"in_#{way}" => processes, &block) # we tried threads but that blew up with weird errors when having lot's of uploads :/
|
138
|
+
end
|
139
|
+
|
140
|
+
def log(text)
|
141
|
+
$stderr.puts text if @config[:verbose]
|
127
142
|
end
|
128
143
|
end
|
129
144
|
|
@@ -157,6 +172,7 @@ module S3MetaSync
|
|
157
172
|
opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
|
158
173
|
opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
|
159
174
|
opts.on("-p", "--parallel COUNT", Integer, "Use COUNT processes for download/upload default: 10") { |c| options[:parallel] = c }
|
175
|
+
opts.on("-V", "--verbose", "Verbose mode"){ options[:verbose] = true }
|
160
176
|
opts.on("-h", "--help", "Show this.") { puts opts; exit }
|
161
177
|
opts.on("-v", "--version", "Show Version"){ puts VERSION; exit}
|
162
178
|
end.parse!(argv)
|
data.tar.gz.sig
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
�
|
2
|
+
l�.N�!�hD��j���������N���7$u+�D��=��ݤ&Hl��$6���^���/v�q<�n�L����rj�0�^����6*:2�\��Ѿe��An�{��-f^<��������uUe��f�0����V�g�_�&(w˦���(JGU���,�(�=O������6��!�*3_+rr��.�������}GT�Ԯ0����w��R���z!�cB��%G�M����ɿ��r�2a�heV���8�
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_meta_sync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
@@ -29,7 +29,7 @@ cert_chain:
|
|
29
29
|
y0kCSWmK6D+x/SbfS6r7Ke07MRqziJdB9GuE1+0cIRuFh8EQ+LN6HXCKM5pon/GU
|
30
30
|
ycwMXfl0
|
31
31
|
-----END CERTIFICATE-----
|
32
|
-
date: 2013-09-
|
32
|
+
date: 2013-09-27 00:00:00.000000000 Z
|
33
33
|
dependencies:
|
34
34
|
- !ruby/object:Gem::Dependency
|
35
35
|
name: aws-sdk
|
metadata.gz.sig
CHANGED
Binary file
|