s3_zipper 1.0.7 → 1.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46522558f243e0d37eaa1aa8d4c2674c563e1d2d83141a05bb714347e7eb30ef
-  data.tar.gz: d0119a21ac771ecc6475f905ba84024ae663745a39dde237a7d2b44f1412855d
+  metadata.gz: 69878c0fb5a4fb5db952cb06bd39ecbb5496cc90eb31fa077d79f78287d3dcab
+  data.tar.gz: 269235d7dbba3e2fabab96d7c86def16db1d440f88c6f73a1f12b3e5f94c1bbd
 SHA512:
-  metadata.gz: 408ff35a601c080805e30165baa10560664bc94b58e4c0f88fc9e0473ab58e9e0e6b9b6f968532fdaea04a23e61e0c67f8425ed593ed2a63fb358556ca630471
-  data.tar.gz: 0f28fac746accbfff9d4d79469ca43cdf850191e0ad05358af967a4336a864adf9eaf4cec363a93e264108fad03c52bee4532a74711c2be3bee73d55ed342781
+  metadata.gz: 51cebd463cf8b28e52ea8b886dbd1ef781e9660ad2bd3d91d75bf3a5b00aafc10348aad6105959167eb9de7265ad2d3a1c6407035b60c0be425aeb6ef8adc055
+  data.tar.gz: ca83241c626075f6e2ec7ed67c27a50b99717ce66d384f9b6937cb0e94e20acc4835c8eb89583b536abe89634a6cf8c596db13afc48e4743a0260af386d39fa9

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    s3_zipper (1.0.7)
+    s3_zipper (1.0.8)
       aws-sdk-s3 (~> 1)
       concurrent-ruby (~> 1.1)
+      multiblock (~> 0.2.1)
       ruby-progressbar (~> 1)
       rubyzip (>= 1.0.0)
 
@@ -12,8 +13,8 @@ GEM
   specs:
     ast (2.4.0)
     aws-eventstream (1.0.3)
-    aws-partitions (1.169.0)
-    aws-sdk-core (3.54.0)
+    aws-partitions (1.170.0)
+    aws-sdk-core (3.54.1)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.1)
@@ -33,6 +34,7 @@ GEM
     jaro_winkler (1.5.2)
     jmespath (1.4.0)
     json (2.2.0)
+    multiblock (0.2.1)
     parallel (1.17.0)
     parser (2.6.3.0)
       ast (~> 2.4.0)

data/README.md

@@ -46,6 +46,25 @@ zipper.zip_to_s3(keys)
 # }
 ```
 
+### Event Handling
+```ruby
+keys = ["documents/files/790/306/985/original/background-10.jpg", "documents/files/790/307/076/original/background-10.jpg"]
+zipper.zip_to_s3 keys do
+  on.start { puts 'Starting to zip' }
+  on.progress { |progress| puts progress.percentage }
+  on.finish { puts 'Finished zipping' }
+  on.upload { puts 'Upload complete' }
+end
+# {
+#   :key=>"3dc29e9ba0a069eb5d0783f07b12e1b3.zip", 
+#   :url => "https://bucket_name.s3.us-west-2.amazonaws.com/35b6f0e2ee91aa0e3c0640c7a4b2b7db.zip" 
+#   :zipped=>["documents/files/790/306/985/original/background-10.jpg", "documents/files/790/307/076/original/background-10.jpg"], 
+#   :failed=>[]
+# }
+```
+
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -54,7 +73,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/s3_zipper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/capshareinc/s3_zipper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 

data/lib/s3_zipper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class S3Zipper
-  VERSION = "1.0.7"
+  VERSION = "1.0.8"
 end

data/lib/s3_zipper.rb

@@ -5,9 +5,10 @@ require "s3_zipper/progress"
 require "s3_zipper/spinner"
 require "s3_zipper/client"
 require "zip"
+require "multiblock"
 
 class S3Zipper
-  attr_accessor :client, :options, :progress, :zip_client
+  attr_accessor :client, :options, :progress, :zip_client, :wrapper
 
   # @param [String] bucket - bucket that files exist in
   # @param [Hash] options - options for zipper
@@ -15,6 +16,7 @@ class S3Zipper
   # @return [S3Zipper]
   def initialize bucket, options = {}
     @options    = options
+    @wrapper    = Multiblock.wrapper
     @progress   = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: nil, length: 80, autofinish: false)
     @client     = Client.new(bucket, options)
     @zip_client = Client.new(options[:zip_bucket], options) if options[:zip_bucket]
@@ -25,18 +27,20 @@ class S3Zipper
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String, File] file - Filename or file object for the zip, defaults to a random string
   # @return [Hash]
-  def zip_to_local_file keys, file: SecureRandom.hex, &block
+  def zip_to_local_file keys, file: SecureRandom.hex
+    yield(wrapper) if block_given?
     file = file.is_a?(File) ? file : File.open("#{file}.zip", "w")
-    zip(keys, file.path, &block)
+    zip(keys, file.path)
   end
 
   # Zips files from s3 to a temporary zip
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String, File] filename - Name of file, defaults to a random string
   # @return [Hash]
-  def zip_to_tempfile keys, filename: SecureRandom.hex, cleanup: false, &block
+  def zip_to_tempfile keys, filename: SecureRandom.hex, cleanup: false
+    yield(wrapper) if block_given?
     zipfile = Tempfile.new([filename, ".zip"])
-    result  = zip(keys, zipfile.path, &block)
+    result  = zip(keys, zipfile.path)
     zipfile.unlink if cleanup
     result
   end
@@ -46,17 +50,29 @@ class S3Zipper
   # @param [String, File] filename - Name of file, defaults to a random string
   # @param [String] path - path for file in s3
   # @return [Hash]
-  def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}, &block
+  def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}
+    yield(wrapper) if block_given?
     filename = "#{path ? "#{path}/" : ''}#{filename}.zip"
-    result   = zip_to_tempfile(keys, filename: filename, cleanup: false, &block)
+    result   = zip_to_tempfile(keys, filename: filename, cleanup: false)
     zip_client.upload(result.delete(:filename), filename, options: s3_options)
     result[:key] = filename
     result[:url] = client.get_url(result[:key])
+    wrapper.call(:upload, result)
     result
   end
 
   private
 
+  def add_to_zip zipfile, filename, file, n = 0
+    existing_file = zipfile.find_entry(filename)
+    if existing_file
+      filename = "#{File.basename(filename, ".*").split('(').first}(#{n})#{File.extname(filename)}"
+      add_to_zip(zipfile, filename, file, n + 1)
+    else
+      zipfile.add(filename, file.path)
+    end
+  end
+
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String] path - path to zip
   # @yield [progress]
@@ -64,15 +80,16 @@ class S3Zipper
   def zip keys, path
     progress.reset total: keys.size, title: "Zipping Keys to #{path}"
     Zip::File.open(path, Zip::File::CREATE) do |zipfile|
+      wrapper.call(:start, zipfile)
       @failed, @successful = client.download_keys keys do |file, key|
         progress.increment title: "Zipping #{key} to #{path}"
-        yield(zipfile, progress) if block_given?
+        wrapper.call(:progress, progress)
         next if file.nil?
-
-        zipfile.add(File.basename(key), file.path)
+        add_to_zip(zipfile, File.basename(key), file)
       end
+      progress.finish(title: "Zipped keys to #{path}")
+      wrapper.call(:finish, zipfile)
     end
-    progress.finish(title: "Zipped keys to #{path}")
     @successful.each { |_, temp| temp.unlink }
     {
       filename: path,

data/s3_zipper.gemspec

@@ -45,6 +45,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "thor"
   spec.add_dependency "aws-sdk-s3", "~> 1"
   spec.add_dependency "concurrent-ruby", "~> 1.1"
+  spec.add_dependency "multiblock", "~> 0.2.1"
   spec.add_dependency "ruby-progressbar", "~> 1"
   spec.add_dependency "rubyzip", ">= 1.0.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_zipper
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.8
 platform: ruby
 authors:
 - Capshare
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-30 00:00:00.000000000 Z
+date: 2019-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -137,6 +137,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: multiblock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement