s3_zipper 1.0.6 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c6bba2b25177fd9baca16cb08efb190bef22a48d25e77650d02c337b3269b85
-  data.tar.gz: 559d7065fbd91f8f59514abf6ebee225b6a7253f478ff02ffe396f31e1000782
+  metadata.gz: f8b6be13c102a9439ad98cbcc6d500cccb7aa06b6e8c354c5c40226ecf876747
+  data.tar.gz: 50e792b49b08fc2545473f8dcf5ad024b1daf1c3d6b7fce155e6ddc8250b3cb4
 SHA512:
-  metadata.gz: 2bdc7b07337e40797e67da68fe1a5607f7d9df367024f275303f886879592360d49e143782ba9d19e78dc072a6d3dde8ebd67b19d578fdfe6ddb5871a39b261c
-  data.tar.gz: ddcbcfc950c3584fd8df1981ca41cb286609d5892f5816b7f027d0b9afd350676a8bbb266369c80ed9d508febc955519159879492bbf94025292e0ebac80afa3
+  metadata.gz: 477f51fb433929dc652e9368a0509c3f4f9ad22d7bd1bec939d04c6be54ce044092da995eb72b6c38a312314dbfb2f3203988b4091e2bc8682ec7d52dee33376
+  data.tar.gz: 36c022cb8a6752f5f1b89a0d2ce9043bc7dc5dd2250c4379b95e37dd13e32decea0eb5d98bf3d6122b697c81349bf7a8042c4b3922a13f4e58e92cd270988395

data/.env.test

@@ -1,4 +1,3 @@
-MOL_TRANSPORTER=redis://localhost
 # AWS KEYS
 AWS_ACCESS_KEY_ID=test-key-id
 AWS_SECRET_ACCESS_KEY=test-secret-key

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    s3_zipper (1.0.6)
+    s3_zipper (1.0.9)
       aws-sdk-s3 (~> 1)
       concurrent-ruby (~> 1.1)
+      multiblock (~> 0.2.1)
       ruby-progressbar (~> 1)
       rubyzip (>= 1.0.0)
 
@@ -12,8 +13,8 @@ GEM
   specs:
     ast (2.4.0)
     aws-eventstream (1.0.3)
-    aws-partitions (1.167.0)
-    aws-sdk-core (3.53.1)
+    aws-partitions (1.171.0)
+    aws-sdk-core (3.54.2)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.1)
@@ -21,10 +22,10 @@ GEM
     aws-sdk-kms (1.21.0)
       aws-sdk-core (~> 3, >= 3.53.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.40.0)
+    aws-sdk-s3 (1.41.0)
       aws-sdk-core (~> 3, >= 3.53.0)
       aws-sdk-kms (~> 1)
-      aws-sigv4 (~> 1.0)
+      aws-sigv4 (~> 1.1)
     aws-sigv4 (1.1.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
     concurrent-ruby (1.1.5)
@@ -33,6 +34,7 @@ GEM
     jaro_winkler (1.5.2)
     jmespath (1.4.0)
     json (2.2.0)
+    multiblock (0.2.1)
     parallel (1.17.0)
     parser (2.6.3.0)
       ast (~> 2.4.0)

data/README.md

@@ -46,6 +46,25 @@ zipper.zip_to_s3(keys)
 # }
 ```
 
+### Event Handling
+```ruby
+keys = ["documents/files/790/306/985/original/background-10.jpg", "documents/files/790/307/076/original/background-10.jpg"]
+zipper.zip_to_s3 keys do
+  on.start { puts 'Starting to zip' }
+  on.progress { |progress| puts progress.percentage }
+  on.finish { puts 'Finished zipping' }
+  on.upload { puts 'Upload complete' }
+end
+# {
+#   :key=>"3dc29e9ba0a069eb5d0783f07b12e1b3.zip", 
+#   :url => "https://bucket_name.s3.us-west-2.amazonaws.com/35b6f0e2ee91aa0e3c0640c7a4b2b7db.zip" 
+#   :zipped=>["documents/files/790/306/985/original/background-10.jpg", "documents/files/790/307/076/original/background-10.jpg"], 
+#   :failed=>[]
+# }
+```
+
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -54,7 +73,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/s3_zipper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/capshareinc/s3_zipper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 

data/lib/s3_zipper.rb

@@ -5,9 +5,10 @@ require "s3_zipper/progress"
 require "s3_zipper/spinner"
 require "s3_zipper/client"
 require "zip"
+require "multiblock"
 
 class S3Zipper
-  attr_accessor :client, :options, :progress, :zip_client
+  attr_accessor :client, :options, :progress, :zip_client, :wrapper
 
   # @param [String] bucket - bucket that files exist in
   # @param [Hash] options - options for zipper
@@ -15,6 +16,7 @@ class S3Zipper
   # @return [S3Zipper]
   def initialize bucket, options = {}
     @options    = options
+    @wrapper    = Multiblock.wrapper
     @progress   = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: nil, length: 80, autofinish: false)
     @client     = Client.new(bucket, options)
     @zip_client = Client.new(options[:zip_bucket], options) if options[:zip_bucket]
@@ -25,18 +27,20 @@ class S3Zipper
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String, File] file - Filename or file object for the zip, defaults to a random string
   # @return [Hash]
-  def zip_to_local_file keys, file: SecureRandom.hex, &block
+  def zip_to_local_file keys, file: SecureRandom.hex
+    yield(wrapper) if block_given?
     file = file.is_a?(File) ? file : File.open("#{file}.zip", "w")
-    zip(keys, file.path, &block)
+    zip(keys, file.path)
   end
 
   # Zips files from s3 to a temporary zip
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String, File] filename - Name of file, defaults to a random string
   # @return [Hash]
-  def zip_to_tempfile keys, filename: SecureRandom.hex, cleanup: false, &block
+  def zip_to_tempfile keys, filename: SecureRandom.hex, cleanup: false
+    yield(wrapper) if block_given?
     zipfile = Tempfile.new([filename, ".zip"])
-    result  = zip(keys, zipfile.path, &block)
+    result  = zip(keys, zipfile.path)
     zipfile.unlink if cleanup
     result
   end
@@ -46,17 +50,29 @@ class S3Zipper
   # @param [String, File] filename - Name of file, defaults to a random string
   # @param [String] path - path for file in s3
   # @return [Hash]
-  def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}, &block
+  def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}
+    yield(wrapper) if block_given?
     filename = "#{path ? "#{path}/" : ''}#{filename}.zip"
-    result   = zip_to_tempfile(keys, filename: filename, cleanup: false, &block)
+    result   = zip_to_tempfile(keys, filename: filename, cleanup: false)
     zip_client.upload(result.delete(:filename), filename, options: s3_options)
     result[:key] = filename
-    result[:url] = client.get_url(result[:key])
+    result[:url] = zip_client.get_url(result[:key])
+    wrapper.call(:upload, result)
     result
   end
 
   private
 
+  def add_to_zip zipfile, filename, file, n = 0
+    existing_file = zipfile.find_entry(filename)
+    if existing_file
+      filename = "#{File.basename(filename, ".*").split('(').first}(#{n})#{File.extname(filename)}"
+      add_to_zip(zipfile, filename, file, n + 1)
+    else
+      zipfile.add(filename, file.path)
+    end
+  end
+
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String] path - path to zip
   # @yield [progress]
@@ -64,15 +80,16 @@ class S3Zipper
   def zip keys, path
     progress.reset total: keys.size, title: "Zipping Keys to #{path}"
     Zip::File.open(path, Zip::File::CREATE) do |zipfile|
+      wrapper.call(:start, zipfile)
       @failed, @successful = client.download_keys keys do |file, key|
         progress.increment title: "Zipping #{key} to #{path}"
-        yield(zipfile, progress) if block_given?
+        wrapper.call(:progress, progress)
         next if file.nil?
-
-        zipfile.add(File.basename(key), file.path)
+        add_to_zip(zipfile, File.basename(key), file)
       end
+      progress.finish(title: "Zipped keys to #{path}")
+      wrapper.call(:finish, zipfile)
     end
-    progress.finish(title: "Zipped keys to #{path}")
     @successful.each { |_, temp| temp.unlink }
     {
       filename: path,

data/lib/s3_zipper/spinner.rb

@@ -6,13 +6,13 @@ class S3Zipper
   class Spinner
     include Concurrent::Async
 
-    def initialize title: "", enabled: true, steps: %w[⎺ ⎻ ⎼ ⎽ ⎼ ⎻]
+    def initialize title: "", enabled: true, steps: %w[▸▹▹▹▹▹ ▹▸▹▹▹▹ ▹▹▸▹▹▹ ▹▹▹▸▹▹ ▹▹▹▹▸▹ ▹▹▹▹▹▸ ▹▹▹▹▹]
       return unless enabled || true
 
       @progress_bar = ProgressBar.create(
-        format:                           "[%i] %t",
+        format:                           "[%B] %t",
         total:                            nil,
-        length:                           97,
+        length:                           100,
         title:                            title,
         autofinish:                       false,
         unknown_progress_animation_steps: steps,
@@ -37,6 +37,7 @@ class S3Zipper
       return unless @progress_bar
       until @progress_bar.finished?
         increment
+        sleep(2)
       end
     end
 

data/lib/s3_zipper/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class S3Zipper
-  VERSION = "1.0.6"
+  VERSION = "3.0.0"
 end

data/s3_zipper.gemspec

@@ -45,6 +45,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "thor"
   spec.add_dependency "aws-sdk-s3", "~> 1"
   spec.add_dependency "concurrent-ruby", "~> 1.1"
+  spec.add_dependency "multiblock", "~> 0.2.1"
   spec.add_dependency "ruby-progressbar", "~> 1"
   spec.add_dependency "rubyzip", ">= 1.0.0"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: s3_zipper
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 3.0.0
 platform: ruby
 authors:
 - Capshare
 - Nickolas Komarnitsky
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-05-30 00:00:00.000000000 Z
+date: 2021-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -137,6 +137,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: multiblock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -202,7 +216,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/capshareinc/s3zipper
   source_code_uri: https://github.com/capshareinc/s3zipper
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -217,9 +231,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: Gem for zipping files in s3
 test_files: []