s3_meta_sync 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/lib/s3_meta_sync.rb +16 -3
- data/lib/s3_meta_sync/version.rb +1 -1
- metadata +1 -1
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 37a24cec2c4aebc13b052d615ef89e90680f2382
|
4
|
+
data.tar.gz: 27acabc2a3fa0a110dad04c1567c78fc37d7ed07
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 003a7fe86ed7329f5352540790edb64168bbe6fe7e6d51e6e4539ee8238744c4fd3e9bfa6e96aa81fe7c23a8b4d762ed94595453c2860930c9cd36a5ddaeccde
|
7
|
+
data.tar.gz: 2b6d7b57bb944b4ad31316a400323223ee7a32d86df96d622b3bb0d14f941e90618399d8ade1d9faa6ea87f931b4828070a2a67ce03eb1054d51a590ab64c462
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data.tar.gz.sig
CHANGED
Binary file
|
data/lib/s3_meta_sync.rb
CHANGED
@@ -37,7 +37,7 @@ module S3MetaSync
|
|
37
37
|
generate_meta(source)
|
38
38
|
local_info = read_meta(source)
|
39
39
|
|
40
|
-
local_info
|
40
|
+
in_multiple_threads(local_info) do |path, md5|
|
41
41
|
next if remote_info[path] == md5
|
42
42
|
upload_file(source, path, destination)
|
43
43
|
end
|
@@ -90,7 +90,7 @@ module S3MetaSync
|
|
90
90
|
content = download_content("#{source}/#{path}")
|
91
91
|
file = "#{destination}/#{path}"
|
92
92
|
FileUtils.mkdir_p(File.dirname(file))
|
93
|
-
File.write(file, content)
|
93
|
+
File.write(file, content, :encoding => content.encoding)
|
94
94
|
end
|
95
95
|
|
96
96
|
def download_content(path)
|
@@ -105,7 +105,7 @@ module S3MetaSync
|
|
105
105
|
generate_meta(destination)
|
106
106
|
local_info = read_meta(destination) # TODO maybe generate !?
|
107
107
|
|
108
|
-
remote_info
|
108
|
+
in_multiple_threads(remote_info) do |path, md5|
|
109
109
|
next if local_info[path] == md5
|
110
110
|
download_file(source, path, destination)
|
111
111
|
end
|
@@ -122,6 +122,18 @@ module S3MetaSync
|
|
122
122
|
def region
|
123
123
|
@config[:region] unless @config[:region].to_s.empty?
|
124
124
|
end
|
125
|
+
|
126
|
+
def in_multiple_threads(data, &block)
|
127
|
+
threads = [@config[:parallel] || 10, data.size].min
|
128
|
+
data = data.to_a
|
129
|
+
(1..threads).to_a.map do
|
130
|
+
Thread.new do
|
131
|
+
while chunk = data.pop
|
132
|
+
yield chunk
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end.map(&:join)
|
136
|
+
end
|
125
137
|
end
|
126
138
|
|
127
139
|
class << self
|
@@ -149,6 +161,7 @@ module S3MetaSync
|
|
149
161
|
opts.on("-k", "--key KEY", "AWS access key") { |c| options[:key] = c }
|
150
162
|
opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
|
151
163
|
opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
|
164
|
+
opts.on("-p", "--parallel COUNT", Integer, "Use COUNT threads for download/upload default: 10") { |c| options[:parallel] = c }
|
152
165
|
opts.on("-h", "--help", "Show this.") { puts opts; exit }
|
153
166
|
opts.on("-v", "--version", "Show Version"){ puts VERSION; exit}
|
154
167
|
end.parse!(argv)
|
data/lib/s3_meta_sync/version.rb
CHANGED
metadata
CHANGED
metadata.gz.sig
CHANGED
Binary file
|