s3_zipper 1.0.5 → 2.0.0

data/Gemfile.lock

@@ -1,34 +1,38 @@
 PATH
   remote: .
   specs:
-    s3_zipper (1.0.4)
-      aws-sdk-s3 (~> 1)
+    s3_zipper (1.0.9)
+      aws-sdk (~> 2)
+      concurrent-ruby (~> 1.1)
+      multiblock (~> 0.2.1)
       ruby-progressbar (~> 1)
       rubyzip (>= 1.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-eventstream (1.0.3)
-    aws-partitions (1.159.0)
-    aws-sdk-core (3.50.0)
-      aws-eventstream (~> 1.0, >= 1.0.2)
-      aws-partitions (~> 1.0)
-      aws-sigv4 (~> 1.1)
-      jmespath (~> 1.0)
-    aws-sdk-kms (1.18.0)
-      aws-sdk-core (~> 3, >= 3.48.2)
-      aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.36.1)
-      aws-sdk-core (~> 3, >= 3.48.2)
-      aws-sdk-kms (~> 1)
+    ast (2.4.0)
+    aws-eventstream (1.1.0)
+    aws-sdk (2.11.632)
+      aws-sdk-resources (= 2.11.632)
+    aws-sdk-core (2.11.632)
       aws-sigv4 (~> 1.0)
-    aws-sigv4 (1.1.0)
-      aws-eventstream (~> 1.0, >= 1.0.2)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.11.632)
+      aws-sdk-core (= 2.11.632)
+    aws-sigv4 (1.2.2)
+      aws-eventstream (~> 1, >= 1.0.2)
+    concurrent-ruby (1.1.5)
     diff-lcs (1.3)
     docile (1.3.1)
+    jaro_winkler (1.5.2)
     jmespath (1.4.0)
     json (2.2.0)
+    multiblock (0.2.1)
+    parallel (1.17.0)
+    parser (2.6.3.0)
+      ast (~> 2.4.0)
+    rainbow (3.0.0)
     rake (10.5.0)
     rake-compiler (1.0.7)
       rake
@@ -45,14 +49,22 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
+    rubocop (0.70.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.6)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
     ruby-progressbar (1.10.0)
-    rubyzip (1.2.2)
+    rubyzip (1.2.3)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
     thor (0.20.3)
+    unicode-display_width (1.6.0)
 
 PLATFORMS
   ruby
@@ -62,9 +74,10 @@ DEPENDENCIES
   rake (~> 10.0)
   rake-compiler
   rspec (~> 3.0)
+  rubocop (~> 0.70.0)
   s3_zipper!
   simplecov
   thor
 
 BUNDLED WITH
-   2.0.1
+   2.2.12

data/README.md

@@ -46,6 +46,25 @@ zipper.zip_to_s3(keys)
 # }
 ```
 
+### Event Handling
+```ruby
+keys = ["documents/files/790/306/985/original/background-10.jpg", "documents/files/790/307/076/original/background-10.jpg"]
+zipper.zip_to_s3 keys do
+  on.start { puts 'Starting to zip' }
+  on.progress { |progress| puts progress.percentage }
+  on.finish { puts 'Finished zipping' }
+  on.upload { puts 'Upload complete' }
+end
+# {
+#   :key=>"3dc29e9ba0a069eb5d0783f07b12e1b3.zip", 
+#   :url => "https://bucket_name.s3.us-west-2.amazonaws.com/35b6f0e2ee91aa0e3c0640c7a4b2b7db.zip" 
+#   :zipped=>["documents/files/790/306/985/original/background-10.jpg", "documents/files/790/307/076/original/background-10.jpg"], 
+#   :failed=>[]
+# }
+```
+
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -54,7 +73,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/s3_zipper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/capshareinc/s3_zipper. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 

data/bin/console

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require "bundler/setup"
 require "s3_zipper"

data/exe/s3_zipper

@@ -1,24 +1,25 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require "s3_zipper"
-require 'thor'
-require 'awesome_print'
+require "thor"
+require "awesome_print"
 class CLI < Thor
-  desc 'local_file FILES', 'zip files from s3 to a local file'
-  method_option :filename, type: :string, aliases: 'f', desc: 'Name of the created zip archive'
-  method_option :bucket, type: :string, default: ENV['AWS_BUCKET'], required: !ENV['AWS_BUCKET'].nil?, aliases: 'b', desc: 'Name of the bucket the files are in'
+  desc "local_file FILES", "zip files from s3 to a local file"
+  method_option :filename, type: :string, aliases: "f", desc: "Name of the created zip archive"
+  method_option :bucket, type: :string, default: ENV["AWS_BUCKET"], required: !ENV["AWS_BUCKET"].nil?, aliases: "b", desc: "Name of the bucket the files are in"
 
-  def local_file(*files)
+  def local_file *files
     S3Zipper.new(options[:bucket]).zip_to_local_file(files, file: options[:filename] || SecureRandom.hex)
   end
 
-  desc 's3 FILES', 'zip files from s3 to s3'
-  method_option :filename, type: :string, aliases: 'f', desc: 'Name of the created zip archive'
-  method_option :path, type: :string, aliases: 'p', desc: 'Path to the file in s3'
-  method_option :bucket, type: :string, default: ENV['AWS_BUCKET'], required: !ENV['AWS_BUCKET'].nil?, aliases: 'b', desc: 'Name of the bucket the files are in'
+  desc "s3 FILES", "zip files from s3 to s3"
+  method_option :filename, type: :string, aliases: "f", desc: "Name of the created zip archive"
+  method_option :path, type: :string, aliases: "p", desc: "Path to the file in s3"
+  method_option :bucket, type: :string, default: ENV["AWS_BUCKET"], required: !ENV["AWS_BUCKET"].nil?, aliases: "b", desc: "Name of the bucket the files are in"
 
-  def s3(*files)
+  def s3 *files
     S3Zipper.new(options[:bucket]).zip_to_s3(files, filename: options[:filename] || SecureRandom.hex, path: options[:path])
   end
 end
-CLI.start(ARGV)
+CLI.start(ARGV)

data/lib/s3_zipper.rb

@@ -1,37 +1,46 @@
+# frozen_string_literal: true
+
 require "s3_zipper/version"
-require 's3_zipper/progress'
+require "s3_zipper/progress"
+require "s3_zipper/spinner"
 require "s3_zipper/client"
 require "zip"
+require "multiblock"
 
 class S3Zipper
-  attr_accessor :client, :options, :progress
+  attr_accessor :client, :options, :progress, :zip_client, :wrapper
 
   # @param [String] bucket - bucket that files exist in
   # @param [Hash] options - options for zipper
   # @option options [Boolean] :progress - toggles progress tracking
   # @return [S3Zipper]
   def initialize bucket, options = {}
-    @options  = options
-    @client   = Client.new(bucket, options)
-    @progress = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: nil, length: 80, autofinish: false)
+    @options    = options
+    @wrapper    = Multiblock.wrapper
+    @progress   = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: nil, length: 80, autofinish: false)
+    @client     = Client.new(bucket, options)
+    @zip_client = Client.new(options[:zip_bucket], options) if options[:zip_bucket]
+    @zip_client ||= @client
   end
 
   # Zips files from s3 to a local zip
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String, File] file - Filename or file object for the zip, defaults to a random string
   # @return [Hash]
-  def zip_to_local_file(keys, file: SecureRandom.hex, &block)
-    file = file.is_a?(File) ? file : File.open("#{file}.zip", 'w')
-    zip(keys, file.path, &block)
+  def zip_to_local_file keys, file: SecureRandom.hex
+    yield(wrapper) if block_given?
+    file = file.is_a?(File) ? file : File.open("#{file}.zip", "w")
+    zip(keys, file.path)
   end
 
   # Zips files from s3 to a temporary zip
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String, File] filename - Name of file, defaults to a random string
   # @return [Hash]
-  def zip_to_tempfile(keys, filename: SecureRandom.hex, cleanup: false, &block)
-    zipfile = Tempfile.new([filename, '.zip'])
-    result  = zip(keys, zipfile.path, &block)
+  def zip_to_tempfile keys, filename: SecureRandom.hex, cleanup: false
+    yield(wrapper) if block_given?
+    zipfile = Tempfile.new([filename, ".zip"])
+    result  = zip(keys, zipfile.path)
     zipfile.unlink if cleanup
     result
   end
@@ -41,36 +50,45 @@ class S3Zipper
   # @param [String, File] filename - Name of file, defaults to a random string
   # @param [String] path - path for file in s3
   # @return [Hash]
-  def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}, &block
-    progress.update :total, 1
+  def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}
+    yield(wrapper) if block_given?
     filename = "#{path ? "#{path}/" : ''}#{filename}.zip"
-    result   = zip_to_tempfile(keys, filename: filename, cleanup: false, &block)
-    progress.update_attrs title: "Uploading zip to s3", total: nil
-    client.upload(result.delete(:filename), filename, options: s3_options)
-    progress.finish(title: "Uploaded zip to #{filename}")
+    result   = zip_to_tempfile(keys, filename: filename, cleanup: false)
+    zip_client.upload(result.delete(:filename), filename, options: s3_options)
     result[:key] = filename
-    result[:url] = client.get_url(result[:key])
+    result[:url] = zip_client.get_url(result[:key])
+    wrapper.call(:upload, result)
     result
   end
 
   private
 
+  def add_to_zip zipfile, filename, file, n = 0
+    existing_file = zipfile.find_entry(filename)
+    if existing_file
+      filename = "#{File.basename(filename, ".*").split('(').first}(#{n})#{File.extname(filename)}"
+      add_to_zip(zipfile, filename, file, n + 1)
+    else
+      zipfile.add(filename, file.path)
+    end
+  end
+
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String] path - path to zip
   # @yield [progress]
   # @return [Hash]
-  def zip(keys, path)
-    total = progress.total || 0
-    total += keys.size
-    progress.update_attrs total: total, title: "Zipping Keys to #{path}"
+  def zip keys, path
+    progress.reset total: keys.size, title: "Zipping Keys to #{path}"
     Zip::File.open(path, Zip::File::CREATE) do |zipfile|
+      wrapper.call(:start, zipfile)
       @failed, @successful = client.download_keys keys do |file, key|
-        progress.increment
-        progress.update :title, "Zipping #{key} to #{path}"
-        yield(zipfile, progress) if block_given?
+        progress.increment title: "Zipping #{key} to #{path}"
+        wrapper.call(:progress, progress)
         next if file.nil?
-        zipfile.add(File.basename(key), file.path)
+        add_to_zip(zipfile, File.basename(key), file)
       end
+      progress.finish(title: "Zipped keys to #{path}")
+      wrapper.call(:finish, zipfile)
     end
     @successful.each { |_, temp| temp.unlink }
     {

data/lib/s3_zipper/client.rb

@@ -1,4 +1,6 @@
-require 'aws-sdk-s3'
+# frozen_string_literal: true
+
+require "aws-sdk"
 
 class S3Zipper
   class Client
@@ -42,18 +44,26 @@ class S3Zipper
       temp.binmode
       temp = download_to_file key, temp
       return if temp.nil?
+
       yield(temp) if block_given?
       temp
     ensure
       temp&.unlink if cleanup
     end
 
-    def get_url(key)
+    def get_url key
       resource.bucket(bucket_name).object(key).public_url
     end
 
     def upload local_path, repo_path, options: {}
-      client.put_object(options.merge!(bucket: bucket_name, key: repo_path, body: File.open(local_path).read))
+      spinner = Spinner.new(
+        enabled: options[:progress],
+        title:   "Uploading zip to #{bucket_name}/#{repo_path}",
+      )
+      spinner.start
+      object = client.put_object(options.merge!(bucket: bucket_name, key: repo_path, body: File.open(local_path).read))
+      spinner.finish title: "Uploaded zip to #{bucket_name}/#{repo_path}"
+      object
     end
   end
 end

data/lib/s3_zipper/progress.rb

@@ -1,38 +1,61 @@
-require 'ruby-progressbar'
+# frozen_string_literal: true
+
+require "ruby-progressbar"
 class Progress
+
   def initialize options = {}
     return unless options[:enabled] || true
+
     @options      = options
     @format       = options[:format]
     @progress_bar = ProgressBar.create(@options)
   end
 
-  def reset(title: nil, total: nil, format: nil)
+  def reset title: nil, total: nil, format: nil
+    return unless @progress_bar
+
     @progress_bar.progress = 0
-    @progress_bar.title    = title
-    @progress_bar.total    = total
-    @progress_bar.format   = format
+    @progress_bar.title    = title if title
+    @progress_bar.total    = total if total
+    @progress_bar.format   = format if format
     refresh
   end
 
+  def spin
+    until @progress_bar.finished?
+      increment
+    end
+  end
+
   def total
-    @progress_bar&.total
+    return unless @progress_bar
+
+    @progress_bar.total
   end
 
   def percentage
-    @progress_bar&.to_h['percentage']
+    return unless @progress_bar
+
+    @progress_bar.to_h["percentage"]
   end
 
   def refresh
-    @progress_bar&.refresh
+    return unless @progress_bar
+
+    @progress_bar.refresh
   end
 
   def progress
-    @progress_bar&.progress
+    return unless @progress_bar
+
+    @progress_bar.progress
   end
 
-  def increment
-    @progress_bar&.increment
+  def increment attrs = {}
+    return unless @progress_bar
+
+    @progress_bar.increment
+    update_attrs(attrs) unless attrs.empty?
   end
 
   def update_attrs attrs
@@ -41,11 +64,13 @@ class Progress
 
   def update attr, value
     return unless @progress_bar
+
     @progress_bar.send("#{attr}=", value)
   end
 
   def finish title: nil, format: nil
     return unless @progress_bar
+
     @progress_bar.title  = title if title
     @progress_bar.format = format if format
     @progress_bar.finish
@@ -57,6 +82,7 @@ class Progress
 
   def get_attr attr
     return unless @progress_bar
+
     @progress_bar.send(attr)
   end
-end
+end

data/lib/s3_zipper/spinner.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "ruby-progressbar"
+require "concurrent-ruby"
+class S3Zipper
+  class Spinner
+    include Concurrent::Async
+
+    def initialize title: "", enabled: true, steps: %w[▸▹▹▹▹▹ ▹▸▹▹▹▹ ▹▹▸▹▹▹ ▹▹▹▸▹▹ ▹▹▹▹▸▹ ▹▹▹▹▹▸ ▹▹▹▹▹]
+      return unless enabled || true
+
+      @progress_bar = ProgressBar.create(
+        format:                           "[%B] %t",
+        total:                            nil,
+        length:                           100,
+        title:                            title,
+        autofinish:                       false,
+        unknown_progress_animation_steps: steps,
+      )
+    end
+
+    def reset title: nil, total: nil, format: nil
+      return unless @progress_bar
+
+      @progress_bar.progress = 0
+      @progress_bar.title    = title if title
+      @progress_bar.total    = total if total
+      @progress_bar.format   = format if format
+      refresh
+    end
+
+    def start
+      async.spin
+    end
+
+    def spin
+      return unless @progress_bar
+      until @progress_bar.finished?
+        increment
+        sleep(2)
+      end
+    end
+
+    def increment
+      return unless @progress_bar
+
+      @progress_bar.increment
+    end
+
+    def finish title: nil
+      return unless @progress_bar
+
+      @progress_bar.title  = title if title
+      @progress_bar.format = "[✔] %t"
+      @progress_bar.finish
+    end
+  end
+end