s3_zipper 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1df86be6cbf4a8a1faf25f919821e021dbb532663d72ffe5eed7e5093dc1ce0a
-  data.tar.gz: 5d8adcb636fc848363268db0d8ef20b8a8cb9abf7b1c01ddd762ffa0079ca821
+  metadata.gz: 3b95e5aca228b0f86b55927fc078f29983c5bae54fe072a6869f080e4776c970
+  data.tar.gz: 17bb91da63992e252a5f32e0c26a82a71576977c391523466e7f20195b400c89
 SHA512:
-  metadata.gz: 55722eca67823cd8ff98b6638b5526c0e3c5a6f1bd3659e00f4b38c2a68264e459422de0abbbf588fcb24fcc39437f43d1492333237d6baf4457d58bd2d7a007
-  data.tar.gz: 3d10cd54337e44e344a5e3ea047d9f34d48e6fd756528b65dd931e14d801da2921dd577a2eefb5fb65ec8f8e8aa31b368a7ce43890cb453cb13a47f143471c14
+  metadata.gz: c0fbb83faf134737110112b20fb06083a5413f92b12f1a18e549f0604960c6827260e0be7192e5cbf6b7bf318fb0ba5ecf95dbc26d020d369f86c3c34598baa6
+  data.tar.gz: 9c7ab23488ca12a32d7bc4e6b2d3410244c5380856d630d37edb84dc6cd51d94964a4a1485e793817ebde2a74e2e52353b9306fcf47e543ef20c37e43b363e5d

data/Gemfile.lock

@@ -10,13 +10,13 @@ GEM
   remote: https://rubygems.org/
   specs:
     aws-eventstream (1.0.3)
-    aws-partitions (1.156.0)
-    aws-sdk-core (3.49.0)
+    aws-partitions (1.158.0)
+    aws-sdk-core (3.50.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.17.0)
+    aws-sdk-kms (1.18.0)
       aws-sdk-core (~> 3, >= 3.48.2)
       aws-sigv4 (~> 1.1)
     aws-sdk-s3 (1.36.1)

data/lib/s3_zipper.rb

@@ -4,35 +4,34 @@ require "s3_zipper/client"
 require "zip"
 
 class S3Zipper
-  attr_accessor :client, :options
+  attr_accessor :client, :options, :progress
 
   # @param [String] bucket - bucket that files exist in
   # @param [Hash] options - options for zipper
   # @option options [Boolean] :progress - toggles progress tracking
   # @return [S3Zipper]
   def initialize bucket, options = {}
-    @options = options
-    @client  = Client.new(bucket, options)
+    @options  = options
+    @client   = Client.new(bucket, options)
+    @progress = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: nil, length: 80, autofinish: false)
   end
 
   # Zips files from s3 to a local zip
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String, File] file - Filename or file object for the zip, defaults to a random string
   # @return [Hash]
-  def zip_to_local_file(keys, file: SecureRandom.hex)
+  def zip_to_local_file(keys, file: SecureRandom.hex, &block)
     file = file.is_a?(File) ? file : File.open("#{file}.zip", 'w')
-    zip keys, file.path do |progress|
-      yield progress if block_given?
-    end
+    zip(keys, file.path, &block)
   end
 
   # Zips files from s3 to a temporary zip
   # @param [Array] keys - Array of s3 keys to zip
   # @param [String, File] filename - Name of file, defaults to a random string
   # @return [Hash]
-  def zip_to_tempfile(keys, filename: SecureRandom.hex, cleanup: false)
+  def zip_to_tempfile(keys, filename: SecureRandom.hex, cleanup: false, &block)
     zipfile = Tempfile.new([filename, '.zip'])
-    result  = zip(keys, zipfile.path) { |progress| yield(zipfile, progress) if block_given? }
+    result  = zip(keys, zipfile.path, &block)
     zipfile.unlink if cleanup
     result
   end
@@ -42,12 +41,13 @@ class S3Zipper
   # @param [String, File] filename - Name of file, defaults to a random string
   # @param [String] path - path for file in s3
   # @return [Hash]
-  def zip_to_s3 keys, filename: SecureRandom.hex, path: nil
+  def zip_to_s3 keys, filename: SecureRandom.hex, path: nil, s3_options: {}, &block
+    progress.update :total, 1
     filename = "#{path ? "#{path}/" : ''}#{filename}.zip"
-    result   = zip_to_tempfile(keys, filename: filename, cleanup: false) do |_, progress|
-      yield(progress) if block_given?
-    end
-    client.upload(result.delete(:filename), filename)
+    result   = zip_to_tempfile(keys, filename: filename, cleanup: false, &block)
+    progress.update_attrs title: "Uploading zip to s3", total: nil
+    client.upload(result.delete(:filename), filename, options: s3_options)
+    progress.finish(title: "Uploaded zip to #{filename}")
     result[:key] = filename
     result[:url] = client.get_url(result[:key])
     result
@@ -60,23 +60,23 @@ class S3Zipper
   # @yield [progress]
   # @return [Hash]
   def zip(keys, path)
-    failed, successful = client.download_keys keys
+    total = progress.total || 0
+    total += keys.size
+    progress.update_attrs total: total, title: "Zipping Keys to #{path}"
     Zip::File.open(path, Zip::File::CREATE) do |zipfile|
-      pb = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: keys.count, length: 80, autofinish: false)
-      successful.each do |key, file|
-        pb.increment
-        pb.update 'title', "Adding #{key} to #{path}"
+      @failed, @successful = client.download_keys keys do |file, key|
+        progress.increment
+        progress.update :title, "Zipping #{key} to #{path}"
+        yield(zipfile, progress) if block_given?
         next if file.nil?
-        yield(pb.progress) if block_given?
-        zipfile.add(key, file.path)
+        zipfile.add(File.basename(key), file.path)
       end
-      pb.finish(title: "Zipped files to #{path}")
     end
-    successful.each { |_, temp| temp.unlink }
+    @successful.each { |_, temp| temp.unlink }
     {
       filename: path,
-      zipped:   successful.map(&:first),
-      failed:   failed.map(&:first)
+      zipped:   @successful.map(&:first),
+      failed:   @failed.map(&:first)
     }
   end
 end

data/lib/s3_zipper/client.rb

@@ -13,20 +13,15 @@ class S3Zipper
       @client      = options[:client] || ::Aws::S3::Client.new
       @resource    = options[:resource] || ::Aws::S3::Resource.new
       @options     = options
-      @pb          = Progress.new(enabled: options[:progress], format: "%e %c/%C %t", total: nil, length: 80, autofinish: false)
     end
 
-    def download_keys keys
-      pb.reset(total: keys.count, title: 'Downloading Keys', format: "%e %c/%C %t")
+    def download_keys keys, cleanup: false
       keys = keys.map do |key|
-        pb.increment
-        pb.update 'title', "Downloading Key: #{key}"
-        temp = download_to_tempfile(key, cleanup: false)
+        temp = download_to_tempfile(key, cleanup: cleanup)
+        yield(temp, key) if block_given?
         [key, temp]
       end
-      keys = keys.partition { |_, temp| temp.nil? }
-      pb.finish(title: 'Downloaded Keys', format: '%e %c/%C %t')
-      keys
+      keys.partition { |_, temp| temp.nil? }
     end
 
     def download key
@@ -57,11 +52,8 @@ class S3Zipper
       resource.bucket(bucket_name).object(key).public_url
     end
 
-    def upload local_path, repo_path
-      pb     = Progress.new(enabled: options[:progress], format: '%t', title: "Uploading '#{local_path}' to '#{repo_path}'", length: 120)
-      object = client.put_object(bucket: bucket_name, key: repo_path, body: File.open(local_path).read)
-      pb.finish(title: "Uploaded '#{local_path}' to '#{repo_path}'")
-      object
+    def upload local_path, repo_path, options: {}
+      client.put_object(options.merge!(bucket: bucket_name, key: repo_path, body: File.open(local_path).read))
     end
   end
 end

data/lib/s3_zipper/progress.rb

@@ -3,8 +3,6 @@ class Progress
   def initialize options = {}
     return unless options[:enabled] || true
     @options      = options
-    @title        = options[:title]
-    @total        = options[:total]
     @format       = options[:format]
     @progress_bar = ProgressBar.create(@options)
   end
@@ -17,35 +15,48 @@ class Progress
     refresh
   end
 
+  def total
+    @progress_bar&.total
+  end
+
+  def percentage
+    @progress_bar&.to_h['percentage']
+  end
+
   def refresh
-    @progress_bar.refresh
+    @progress_bar&.refresh
   end
 
   def progress
-    return unless @progress_bar
-    @progress_bar.progress
+    @progress_bar&.progress
   end
 
   def increment
-    return unless @progress_bar
-    @progress_bar.increment
+    @progress_bar&.increment
+  end
+
+  def update_attrs attrs
+    attrs.each(&method(:update))
   end
 
   def update attr, value
     return unless @progress_bar
     @progress_bar.send("#{attr}=", value)
-    @progress_bar.refresh
   end
 
-  def finish title: @title, format: @format
+  def finish title: nil, format: nil
     return unless @progress_bar
-    @progress_bar.title  = title
-    @progress_bar.format = format
-    @progress_bar.refresh
+    @progress_bar.title  = title if title
+    @progress_bar.format = format if format
     @progress_bar.finish
   end
 
   def disable
     @progress_bar = nil
   end
+
+  def get_attr attr
+    return unless @progress_bar
+    @progress_bar.send(attr)
+  end
 end

data/lib/s3_zipper/version.rb

@@ -1,3 +1,3 @@
 class S3Zipper
-  VERSION = "1.0.3"
+  VERSION = "1.0.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_zipper
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Capshare
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-01 00:00:00.000000000 Z
+date: 2019-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler