s3_zipper 1.0.6 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.env.test +0 -1
- data/Gemfile.lock +7 -5
- data/README.md +20 -1
- data/lib/s3_zipper.rb +29 -12
- data/lib/s3_zipper/spinner.rb +4 -3
- data/lib/s3_zipper/version.rb +1 -1
- data/s3_zipper.gemspec +1 -0
- metadata +20 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f8b6be13c102a9439ad98cbcc6d500cccb7aa06b6e8c354c5c40226ecf876747
|
4
|
+
data.tar.gz: 50e792b49b08fc2545473f8dcf5ad024b1daf1c3d6b7fce155e6ddc8250b3cb4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 477f51fb433929dc652e9368a0509c3f4f9ad22d7bd1bec939d04c6be54ce044092da995eb72b6c38a312314dbfb2f3203988b4091e2bc8682ec7d52dee33376
|
7
|
+
data.tar.gz: 36c022cb8a6752f5f1b89a0d2ce9043bc7dc5dd2250c4379b95e37dd13e32decea0eb5d98bf3d6122b697c81349bf7a8042c4b3922a13f4e58e92cd270988395
|
data/.env.test
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
s3_zipper (1.0.
|
4
|
+
s3_zipper (1.0.9)
|
5
5
|
aws-sdk-s3 (~> 1)
|
6
6
|
concurrent-ruby (~> 1.1)
|
7
|
+
multiblock (~> 0.2.1)
|
7
8
|
ruby-progressbar (~> 1)
|
8
9
|
rubyzip (>= 1.0.0)
|
9
10
|
|
@@ -12,8 +13,8 @@ GEM
|
|
12
13
|
specs:
|
13
14
|
ast (2.4.0)
|
14
15
|
aws-eventstream (1.0.3)
|
15
|
-
aws-partitions (1.
|
16
|
-
aws-sdk-core (3.
|
16
|
+
aws-partitions (1.171.0)
|
17
|
+
aws-sdk-core (3.54.2)
|
17
18
|
aws-eventstream (~> 1.0, >= 1.0.2)
|
18
19
|
aws-partitions (~> 1.0)
|
19
20
|
aws-sigv4 (~> 1.1)
|
@@ -21,10 +22,10 @@ GEM
|
|
21
22
|
aws-sdk-kms (1.21.0)
|
22
23
|
aws-sdk-core (~> 3, >= 3.53.0)
|
23
24
|
aws-sigv4 (~> 1.1)
|
24
|
-
aws-sdk-s3 (1.
|
25
|
+
aws-sdk-s3 (1.41.0)
|
25
26
|
aws-sdk-core (~> 3, >= 3.53.0)
|
26
27
|
aws-sdk-kms (~> 1)
|
27
|
-
aws-sigv4 (~> 1.
|
28
|
+
aws-sigv4 (~> 1.1)
|
28
29
|
aws-sigv4 (1.1.0)
|
29
30
|
aws-eventstream (~> 1.0, >= 1.0.2)
|
30
31
|
concurrent-ruby (1.1.5)
|
@@ -33,6 +34,7 @@ GEM
|
|
33
34
|
jaro_winkler (1.5.2)
|
34
35
|
jmespath (1.4.0)
|
35
36
|
json (2.2.0)
|
37
|
+
multiblock (0.2.1)
|
36
38
|
parallel (1.17.0)
|
37
39
|
parser (2.6.3.0)
|
38
40
|
ast (~> 2.4.0)
|
data/README.md
CHANGED
@@ -46,6 +46,25 @@ zipper.zip_to_s3(keys)
|
|
46
46
|
# }
|
47
47
|
```
|
48
48
|
|
49
|
+
### Event Handling
|
50
|
+
```ruby
|
51
|
+
keys = ["documents/files/790/306/985/original/background-10.jpg", "documents/files/790/307/076/original/background-10.jpg"]
|
52
|
+
zipper.zip_to_s3 keys do
|
53
|
+
on.start { puts 'Starting to zip' }
|
54
|
+
on.progress { |progress| puts progress.percentage }
|
55
|
+
on.finish { puts 'Finished zipping' }
|
56
|
+
on.upload { puts 'Upload complete' }
|
57
|
+
end
|
58
|
+
# {
|
59
|
+
# :key=>"3dc29e9ba0a069eb5d0783f07b12e1b3.zip",
|
60
|
+
# :url => "https://bucket_name.s3.us-west-2.amazonaws.com/35b6f0e2ee91aa0e3c0640c7a4b2b7db.zip"
|
61
|
+
# :zipped=>["documents/files/790/306/985/original/background-10.jpg", "documents/files/790/307/076/original/background-10.jpg"],
|
62
|
+
# :failed=>[]
|
63
|
+
# }
|
64
|
+
```
|
65
|
+
|
66
|
+
|
67
|
+
|
49
68
|
## Development
|
50
69
|
|
51
70
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
@@ -54,7 +73,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
54
73
|
|
55
74
|
## Contributing
|
56
75
|
|
57
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
76
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/capshareinc/s3_zipper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
58
77
|
|
59
78
|
## License
|
60
79
|
|
data/lib/s3_zipper.rb
CHANGED
@@ -5,9 +5,10 @@ require "s3_zipper/progress"
|
|
5
5
|
require "s3_zipper/spinner"
|
6
6
|
require "s3_zipper/client"
|
7
7
|
require "zip"
|
8
|
+
require "multiblock"
|
8
9
|
|
9
10
|
class S3Zipper
|
10
|
-
attr_accessor :client, :options, :progress, :zip_client
|
11
|
+
attr_accessor :client, :options, :progress, :zip_client, :wrapper
|
11
12
|
|
12
13
|
# @param [String] bucket - bucket that files exist in
|
13
14
|
# @param [Hash] options - options for zipper
|
@@ -15,6 +16,7 @@ class S3Zipper
|
|
15
16
|
# @return [S3Zipper]
|
16
17
|
def initialize bucket, options = {}
|
17
18
|
@options = options
|
19
|
+
@wrapper = Multiblock.wrapper
|
18
20
|
@progress = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: nil, length: 80, autofinish: false)
|
19
21
|
@client = Client.new(bucket, options)
|
20
22
|
@zip_client = Client.new(options[:zip_bucket], options) if options[:zip_bucket]
|
@@ -25,18 +27,20 @@ class S3Zipper
|
|
25
27
|
# @param [Array] keys - Array of s3 keys to zip
|
26
28
|
# @param [String, File] file - Filename or file object for the zip, defaults to a random string
|
27
29
|
# @return [Hash]
|
28
|
-
def zip_to_local_file keys, file: SecureRandom.hex
|
30
|
+
def zip_to_local_file keys, file: SecureRandom.hex
|
31
|
+
yield(wrapper) if block_given?
|
29
32
|
file = file.is_a?(File) ? file : File.open("#{file}.zip", "w")
|
30
|
-
zip(keys, file.path
|
33
|
+
zip(keys, file.path)
|
31
34
|
end
|
32
35
|
|
33
36
|
# Zips files from s3 to a temporary zip
|
34
37
|
# @param [Array] keys - Array of s3 keys to zip
|
35
38
|
# @param [String, File] filename - Name of file, defaults to a random string
|
36
39
|
# @return [Hash]
|
37
|
-
def zip_to_tempfile keys, filename: SecureRandom.hex, cleanup: false
|
40
|
+
def zip_to_tempfile keys, filename: SecureRandom.hex, cleanup: false
|
41
|
+
yield(wrapper) if block_given?
|
38
42
|
zipfile = Tempfile.new([filename, ".zip"])
|
39
|
-
result = zip(keys, zipfile.path
|
43
|
+
result = zip(keys, zipfile.path)
|
40
44
|
zipfile.unlink if cleanup
|
41
45
|
result
|
42
46
|
end
|
@@ -46,17 +50,29 @@ class S3Zipper
|
|
46
50
|
# @param [String, File] filename - Name of file, defaults to a random string
|
47
51
|
# @param [String] path - path for file in s3
|
48
52
|
# @return [Hash]
|
49
|
-
def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}
|
53
|
+
def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}
|
54
|
+
yield(wrapper) if block_given?
|
50
55
|
filename = "#{path ? "#{path}/" : ''}#{filename}.zip"
|
51
|
-
result = zip_to_tempfile(keys, filename: filename, cleanup: false
|
56
|
+
result = zip_to_tempfile(keys, filename: filename, cleanup: false)
|
52
57
|
zip_client.upload(result.delete(:filename), filename, options: s3_options)
|
53
58
|
result[:key] = filename
|
54
|
-
result[:url] =
|
59
|
+
result[:url] = zip_client.get_url(result[:key])
|
60
|
+
wrapper.call(:upload, result)
|
55
61
|
result
|
56
62
|
end
|
57
63
|
|
58
64
|
private
|
59
65
|
|
66
|
+
def add_to_zip zipfile, filename, file, n = 0
|
67
|
+
existing_file = zipfile.find_entry(filename)
|
68
|
+
if existing_file
|
69
|
+
filename = "#{File.basename(filename, ".*").split('(').first}(#{n})#{File.extname(filename)}"
|
70
|
+
add_to_zip(zipfile, filename, file, n + 1)
|
71
|
+
else
|
72
|
+
zipfile.add(filename, file.path)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
60
76
|
# @param [Array] keys - Array of s3 keys to zip
|
61
77
|
# @param [String] path - path to zip
|
62
78
|
# @yield [progress]
|
@@ -64,15 +80,16 @@ class S3Zipper
|
|
64
80
|
def zip keys, path
|
65
81
|
progress.reset total: keys.size, title: "Zipping Keys to #{path}"
|
66
82
|
Zip::File.open(path, Zip::File::CREATE) do |zipfile|
|
83
|
+
wrapper.call(:start, zipfile)
|
67
84
|
@failed, @successful = client.download_keys keys do |file, key|
|
68
85
|
progress.increment title: "Zipping #{key} to #{path}"
|
69
|
-
|
86
|
+
wrapper.call(:progress, progress)
|
70
87
|
next if file.nil?
|
71
|
-
|
72
|
-
zipfile.add(File.basename(key), file.path)
|
88
|
+
add_to_zip(zipfile, File.basename(key), file)
|
73
89
|
end
|
90
|
+
progress.finish(title: "Zipped keys to #{path}")
|
91
|
+
wrapper.call(:finish, zipfile)
|
74
92
|
end
|
75
|
-
progress.finish(title: "Zipped keys to #{path}")
|
76
93
|
@successful.each { |_, temp| temp.unlink }
|
77
94
|
{
|
78
95
|
filename: path,
|
data/lib/s3_zipper/spinner.rb
CHANGED
@@ -6,13 +6,13 @@ class S3Zipper
|
|
6
6
|
class Spinner
|
7
7
|
include Concurrent::Async
|
8
8
|
|
9
|
-
def initialize title: "", enabled: true, steps: %w[
|
9
|
+
def initialize title: "", enabled: true, steps: %w[▸▹▹▹▹▹ ▹▸▹▹▹▹ ▹▹▸▹▹▹ ▹▹▹▸▹▹ ▹▹▹▹▸▹ ▹▹▹▹▹▸ ▹▹▹▹▹]
|
10
10
|
return unless enabled || true
|
11
11
|
|
12
12
|
@progress_bar = ProgressBar.create(
|
13
|
-
format: "[%
|
13
|
+
format: "[%B] %t",
|
14
14
|
total: nil,
|
15
|
-
length:
|
15
|
+
length: 100,
|
16
16
|
title: title,
|
17
17
|
autofinish: false,
|
18
18
|
unknown_progress_animation_steps: steps,
|
@@ -37,6 +37,7 @@ class S3Zipper
|
|
37
37
|
return unless @progress_bar
|
38
38
|
until @progress_bar.finished?
|
39
39
|
increment
|
40
|
+
sleep(2)
|
40
41
|
end
|
41
42
|
end
|
42
43
|
|
data/lib/s3_zipper/version.rb
CHANGED
data/s3_zipper.gemspec
CHANGED
@@ -45,6 +45,7 @@ Gem::Specification.new do |spec|
|
|
45
45
|
spec.add_development_dependency "thor"
|
46
46
|
spec.add_dependency "aws-sdk-s3", "~> 1"
|
47
47
|
spec.add_dependency "concurrent-ruby", "~> 1.1"
|
48
|
+
spec.add_dependency "multiblock", "~> 0.2.1"
|
48
49
|
spec.add_dependency "ruby-progressbar", "~> 1"
|
49
50
|
spec.add_dependency "rubyzip", ">= 1.0.0"
|
50
51
|
end
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_zipper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Capshare
|
8
8
|
- Nickolas Komarnitsky
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2021-04-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -137,6 +137,20 @@ dependencies:
|
|
137
137
|
- - "~>"
|
138
138
|
- !ruby/object:Gem::Version
|
139
139
|
version: '1.1'
|
140
|
+
- !ruby/object:Gem::Dependency
|
141
|
+
name: multiblock
|
142
|
+
requirement: !ruby/object:Gem::Requirement
|
143
|
+
requirements:
|
144
|
+
- - "~>"
|
145
|
+
- !ruby/object:Gem::Version
|
146
|
+
version: 0.2.1
|
147
|
+
type: :runtime
|
148
|
+
prerelease: false
|
149
|
+
version_requirements: !ruby/object:Gem::Requirement
|
150
|
+
requirements:
|
151
|
+
- - "~>"
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
version: 0.2.1
|
140
154
|
- !ruby/object:Gem::Dependency
|
141
155
|
name: ruby-progressbar
|
142
156
|
requirement: !ruby/object:Gem::Requirement
|
@@ -202,7 +216,7 @@ metadata:
|
|
202
216
|
allowed_push_host: https://rubygems.org
|
203
217
|
homepage_uri: https://github.com/capshareinc/s3zipper
|
204
218
|
source_code_uri: https://github.com/capshareinc/s3zipper
|
205
|
-
post_install_message:
|
219
|
+
post_install_message:
|
206
220
|
rdoc_options: []
|
207
221
|
require_paths:
|
208
222
|
- lib
|
@@ -217,9 +231,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
217
231
|
- !ruby/object:Gem::Version
|
218
232
|
version: '0'
|
219
233
|
requirements: []
|
220
|
-
|
221
|
-
|
222
|
-
signing_key:
|
234
|
+
rubygems_version: 3.0.9
|
235
|
+
signing_key:
|
223
236
|
specification_version: 4
|
224
237
|
summary: Gem for zipping files in s3
|
225
238
|
test_files: []
|