s3_zipper 1.0.3 → 1.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -3
- data/lib/s3_zipper.rb +25 -25
- data/lib/s3_zipper/client.rb +6 -14
- data/lib/s3_zipper/progress.rb +23 -12
- data/lib/s3_zipper/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3b95e5aca228b0f86b55927fc078f29983c5bae54fe072a6869f080e4776c970
|
4
|
+
data.tar.gz: 17bb91da63992e252a5f32e0c26a82a71576977c391523466e7f20195b400c89
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c0fbb83faf134737110112b20fb06083a5413f92b12f1a18e549f0604960c6827260e0be7192e5cbf6b7bf318fb0ba5ecf95dbc26d020d369f86c3c34598baa6
|
7
|
+
data.tar.gz: 9c7ab23488ca12a32d7bc4e6b2d3410244c5380856d630d37edb84dc6cd51d94964a4a1485e793817ebde2a74e2e52353b9306fcf47e543ef20c37e43b363e5d
|
data/Gemfile.lock
CHANGED
@@ -10,13 +10,13 @@ GEM
|
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
12
|
aws-eventstream (1.0.3)
|
13
|
-
aws-partitions (1.
|
14
|
-
aws-sdk-core (3.
|
13
|
+
aws-partitions (1.158.0)
|
14
|
+
aws-sdk-core (3.50.0)
|
15
15
|
aws-eventstream (~> 1.0, >= 1.0.2)
|
16
16
|
aws-partitions (~> 1.0)
|
17
17
|
aws-sigv4 (~> 1.1)
|
18
18
|
jmespath (~> 1.0)
|
19
|
-
aws-sdk-kms (1.
|
19
|
+
aws-sdk-kms (1.18.0)
|
20
20
|
aws-sdk-core (~> 3, >= 3.48.2)
|
21
21
|
aws-sigv4 (~> 1.1)
|
22
22
|
aws-sdk-s3 (1.36.1)
|
data/lib/s3_zipper.rb
CHANGED
@@ -4,35 +4,34 @@ require "s3_zipper/client"
|
|
4
4
|
require "zip"
|
5
5
|
|
6
6
|
class S3Zipper
|
7
|
-
attr_accessor :client, :options
|
7
|
+
attr_accessor :client, :options, :progress
|
8
8
|
|
9
9
|
# @param [String] bucket - bucket that files exist in
|
10
10
|
# @param [Hash] options - options for zipper
|
11
11
|
# @option options [Boolean] :progress - toggles progress tracking
|
12
12
|
# @return [S3Zipper]
|
13
13
|
def initialize bucket, options = {}
|
14
|
-
@options
|
15
|
-
@client
|
14
|
+
@options = options
|
15
|
+
@client = Client.new(bucket, options)
|
16
|
+
@progress = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: nil, length: 80, autofinish: false)
|
16
17
|
end
|
17
18
|
|
18
19
|
# Zips files from s3 to a local zip
|
19
20
|
# @param [Array] keys - Array of s3 keys to zip
|
20
21
|
# @param [String, File] file - Filename or file object for the zip, defaults to a random string
|
21
22
|
# @return [Hash]
|
22
|
-
def zip_to_local_file(keys, file: SecureRandom.hex)
|
23
|
+
def zip_to_local_file(keys, file: SecureRandom.hex, &block)
|
23
24
|
file = file.is_a?(File) ? file : File.open("#{file}.zip", 'w')
|
24
|
-
zip
|
25
|
-
yield progress if block_given?
|
26
|
-
end
|
25
|
+
zip(keys, file.path, &block)
|
27
26
|
end
|
28
27
|
|
29
28
|
# Zips files from s3 to a temporary zip
|
30
29
|
# @param [Array] keys - Array of s3 keys to zip
|
31
30
|
# @param [String, File] filename - Name of file, defaults to a random string
|
32
31
|
# @return [Hash]
|
33
|
-
def zip_to_tempfile(keys, filename: SecureRandom.hex, cleanup: false)
|
32
|
+
def zip_to_tempfile(keys, filename: SecureRandom.hex, cleanup: false, &block)
|
34
33
|
zipfile = Tempfile.new([filename, '.zip'])
|
35
|
-
result = zip(keys, zipfile.path
|
34
|
+
result = zip(keys, zipfile.path, &block)
|
36
35
|
zipfile.unlink if cleanup
|
37
36
|
result
|
38
37
|
end
|
@@ -42,12 +41,13 @@ class S3Zipper
|
|
42
41
|
# @param [String, File] filename - Name of file, defaults to a random string
|
43
42
|
# @param [String] path - path for file in s3
|
44
43
|
# @return [Hash]
|
45
|
-
def zip_to_s3 keys, filename: SecureRandom.hex, path: nil
|
44
|
+
def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}, &block
|
45
|
+
progress.update :total, 1
|
46
46
|
filename = "#{path ? "#{path}/" : ''}#{filename}.zip"
|
47
|
-
result = zip_to_tempfile(keys, filename: filename, cleanup: false
|
48
|
-
|
49
|
-
|
50
|
-
|
47
|
+
result = zip_to_tempfile(keys, filename: filename, cleanup: false, &block)
|
48
|
+
progress.update_attrs title: "Uploading zip to s3", total: nil
|
49
|
+
client.upload(result.delete(:filename), filename, options: s3_options)
|
50
|
+
progress.finish(title: "Uploaded zip to #{filename}")
|
51
51
|
result[:key] = filename
|
52
52
|
result[:url] = client.get_url(result[:key])
|
53
53
|
result
|
@@ -60,23 +60,23 @@ class S3Zipper
|
|
60
60
|
# @yield [progress]
|
61
61
|
# @return [Hash]
|
62
62
|
def zip(keys, path)
|
63
|
-
|
63
|
+
total = progress.total || 0
|
64
|
+
total += keys.size
|
65
|
+
progress.update_attrs total: total, title: "Zipping Keys to #{path}"
|
64
66
|
Zip::File.open(path, Zip::File::CREATE) do |zipfile|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
67
|
+
@failed, @successful = client.download_keys keys do |file, key|
|
68
|
+
progress.increment
|
69
|
+
progress.update :title, "Zipping #{key} to #{path}"
|
70
|
+
yield(zipfile, progress) if block_given?
|
69
71
|
next if file.nil?
|
70
|
-
|
71
|
-
zipfile.add(key, file.path)
|
72
|
+
zipfile.add(File.basename(key), file.path)
|
72
73
|
end
|
73
|
-
pb.finish(title: "Zipped files to #{path}")
|
74
74
|
end
|
75
|
-
successful.each { |_, temp| temp.unlink }
|
75
|
+
@successful.each { |_, temp| temp.unlink }
|
76
76
|
{
|
77
77
|
filename: path,
|
78
|
-
zipped: successful.map(&:first),
|
79
|
-
failed: failed.map(&:first)
|
78
|
+
zipped: @successful.map(&:first),
|
79
|
+
failed: @failed.map(&:first)
|
80
80
|
}
|
81
81
|
end
|
82
82
|
end
|
data/lib/s3_zipper/client.rb
CHANGED
@@ -13,20 +13,15 @@ class S3Zipper
|
|
13
13
|
@client = options[:client] || ::Aws::S3::Client.new
|
14
14
|
@resource = options[:resource] || ::Aws::S3::Resource.new
|
15
15
|
@options = options
|
16
|
-
@pb = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: nil, length: 80, autofinish: false)
|
17
16
|
end
|
18
17
|
|
19
|
-
def download_keys keys
|
20
|
-
pb.reset(total: keys.count, title: 'Downloading Keys', format: "%e %c/%C %t")
|
18
|
+
def download_keys keys, cleanup: false
|
21
19
|
keys = keys.map do |key|
|
22
|
-
|
23
|
-
|
24
|
-
temp = download_to_tempfile(key, cleanup: false)
|
20
|
+
temp = download_to_tempfile(key, cleanup: cleanup)
|
21
|
+
yield(temp, key) if block_given?
|
25
22
|
[key, temp]
|
26
23
|
end
|
27
|
-
keys
|
28
|
-
pb.finish(title: 'Downloaded Keys', format: '%e %c/%C %t')
|
29
|
-
keys
|
24
|
+
keys.partition { |_, temp| temp.nil? }
|
30
25
|
end
|
31
26
|
|
32
27
|
def download key
|
@@ -57,11 +52,8 @@ class S3Zipper
|
|
57
52
|
resource.bucket(bucket_name).object(key).public_url
|
58
53
|
end
|
59
54
|
|
60
|
-
def upload local_path, repo_path
|
61
|
-
|
62
|
-
object = client.put_object(bucket: bucket_name, key: repo_path, body: File.open(local_path).read)
|
63
|
-
pb.finish(title: "Uploaded '#{local_path}' to '#{repo_path}'")
|
64
|
-
object
|
55
|
+
def upload local_path, repo_path, options: {}
|
56
|
+
client.put_object(options.merge!(bucket: bucket_name, key: repo_path, body: File.open(local_path).read))
|
65
57
|
end
|
66
58
|
end
|
67
59
|
end
|
data/lib/s3_zipper/progress.rb
CHANGED
@@ -3,8 +3,6 @@ class Progress
|
|
3
3
|
def initialize options = {}
|
4
4
|
return unless options[:enabled] || true
|
5
5
|
@options = options
|
6
|
-
@title = options[:title]
|
7
|
-
@total = options[:total]
|
8
6
|
@format = options[:format]
|
9
7
|
@progress_bar = ProgressBar.create(@options)
|
10
8
|
end
|
@@ -17,35 +15,48 @@ class Progress
|
|
17
15
|
refresh
|
18
16
|
end
|
19
17
|
|
18
|
+
def total
|
19
|
+
@progress_bar&.total
|
20
|
+
end
|
21
|
+
|
22
|
+
def percentage
|
23
|
+
@progress_bar&.to_h['percentage']
|
24
|
+
end
|
25
|
+
|
20
26
|
def refresh
|
21
|
-
@progress_bar
|
27
|
+
@progress_bar&.refresh
|
22
28
|
end
|
23
29
|
|
24
30
|
def progress
|
25
|
-
|
26
|
-
@progress_bar.progress
|
31
|
+
@progress_bar&.progress
|
27
32
|
end
|
28
33
|
|
29
34
|
def increment
|
30
|
-
|
31
|
-
|
35
|
+
@progress_bar&.increment
|
36
|
+
end
|
37
|
+
|
38
|
+
def update_attrs attrs
|
39
|
+
attrs.each(&method(:update))
|
32
40
|
end
|
33
41
|
|
34
42
|
def update attr, value
|
35
43
|
return unless @progress_bar
|
36
44
|
@progress_bar.send("#{attr}=", value)
|
37
|
-
@progress_bar.refresh
|
38
45
|
end
|
39
46
|
|
40
|
-
def finish title:
|
47
|
+
def finish title: nil, format: nil
|
41
48
|
return unless @progress_bar
|
42
|
-
@progress_bar.title = title
|
43
|
-
@progress_bar.format = format
|
44
|
-
@progress_bar.refresh
|
49
|
+
@progress_bar.title = title if title
|
50
|
+
@progress_bar.format = format if format
|
45
51
|
@progress_bar.finish
|
46
52
|
end
|
47
53
|
|
48
54
|
def disable
|
49
55
|
@progress_bar = nil
|
50
56
|
end
|
57
|
+
|
58
|
+
def get_attr attr
|
59
|
+
return unless @progress_bar
|
60
|
+
@progress_bar.send(attr)
|
61
|
+
end
|
51
62
|
end
|
data/lib/s3_zipper/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_zipper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Capshare
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-05-
|
12
|
+
date: 2019-05-08 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|