zip_tricks 4.4.2 → 4.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c690345707197b65846cdbda6096a94321edd088
-  data.tar.gz: 23fb11b83016deacb2d94f6fb7ac8a5a2f8a6821
+  metadata.gz: dd1865613940a7038642206baf6cd58457b87ff5
+  data.tar.gz: 3ebda3c665c9671d9f033d6f768ccdc800d7f2f9
 SHA512:
-  metadata.gz: 854c3bd1a6d7d52f83187f925736f610fcc564a3fa892007b27ca4dc8ace5548d56fc872cf6acec2e6395dc70638b97beda07f29877bbe4562d736839ddea0b6
-  data.tar.gz: 475035d3ec2229b154b264bf1b0e7dc8ada36b008cbfea64323bf8e251d54c553957dadac5fa16b0f322ba8c35bc88ca933243325b72a34d2c9533d9d853dffa
+  metadata.gz: c16f15ab10788d90fe96e61e8ecf20a20eee6a7e94d2d22538c34964d51eb9ee4cc64459dd46997f2b994212944e34e8162d7880ff4d79343c86a5d1ed09fd77
+  data.tar.gz: da1d6e24031365ea67fc2fa0fbf3096f57b007c6ac35449b57b1783eecfa703d530c1c695429335bee252a841da6722f3bca341bd6fc8c5e9e24bf6379edee76

data/.codeclimate.yml

@@ -0,0 +1,7 @@
+engines:
+  rubocop:
+    enabled: true
+    
+ratings:
+   paths:
+   - "**.rb"

data/.gitignore

@@ -9,6 +9,9 @@ rdoc
 doc
 .yardoc
 
+# Rubocop
+rubocop.html
+
 # bundler
 .bundle
 Gemfile.lock
@@ -51,3 +54,6 @@ testing/*.zip
 
 # For rubinius:
 #*.rbc
+
+# Rubocop report
+rubocop.html

data/.rubocop.yml

@@ -0,0 +1,79 @@
+inherit_from: .rubocop_todo.yml
+
+Metrics/LineLength:
+  Max: 120
+
+AllCops:
+  Exclude:
+    - spec/**/*.rb
+    - zip_tricks.gemspec
+    - Gemfile
+    - Rakefile
+    - vendor/**/*
+    - julik_scratchpad/**/*
+
+Style/ClassAndModuleChildren:
+  Enabled:
+    false
+
+# "begin; ... rescue; end " without saying that we only rescue StandardError and below.
+# Can be debated, but I think it is reasonable to expect any Rubyist to know what errors
+# the standard rescue clause covers.
+Lint/RescueWithoutErrorClass:
+  Enabled: false
+
+Lint/AssignmentInCondition:
+  Enabled: false
+
+# We use % all over and nothing ever came out of it
+Style/FormatString:
+  Enabled: false
+
+# We have classes with methods having long bodies. We apply a lot of literal small
+# operations which have to be applied in one locality specifically. Additionally,
+# class length is a bit of a boilerplate metric
+Metrics/ClassLength:
+  Enabled: false
+
+# Idem. We have quite some methods that do a lot of small writes/reads in rapid
+# succession - not because those methods have high cyclomatic complexity, but
+# because they perform a single operation consisting of many sequential small
+# ones.
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Exclude:
+    - examples/rack_application.rb
+    - lib/zip_tricks/file_reader.rb
+    - lib/zip_tricks/remote_io.rb
+    - lib/zip_tricks/streamer.rb
+    - testing/support.rb
+
+Metrics/BlockLength:
+  Exclude:
+    - lib/zip_tricks/file_reader.rb
+
+Layout/MultilineOperationIndentation:
+  Exclude:
+    - lib/zip_tricks/file_reader.rb
+
+Metrics/ParameterLists:
+  Exclude:
+    - lib/zip_tricks/streamer.rb
+
+Style/GlobalVars:
+  Exclude:
+    - testing/generate_test_files.rb
+    - testing/support.rb
+
+# The advice this cop gives varies depending on the Ruby version,
+# so no - I will not be following two conflicting recommendations.
+Style/MutableConstant:
+  Enabled: false
+
+Layout/SpaceInsideHashLiteralBraces:
+  EnforcedStyle: no_space
+
+Style/Alias:
+  EnforcedStyle: prefer_alias_method

data/.rubocop_todo.yml

@@ -0,0 +1,43 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2017-09-11 13:38:55 +0200 using RuboCop version 0.49.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 6
+Lint/UselessAssignment:
+  Exclude:
+    - 'lib/zip_tricks/streamer.rb'
+    - 'spec/zip_tricks/block_deflate_spec.rb'
+    - 'spec/zip_tricks/file_reader_spec.rb'
+    - 'spec/zip_tricks/remote_uncap_spec.rb'
+    - 'spec/zip_tricks/stream_crc32_spec.rb'
+    - 'spec/zip_tricks/streamer_spec.rb'
+
+# Offense count: 1
+Metrics/PerceivedComplexity:
+  Max: 8
+
+# Offense count: 4
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/zip_tricks/block_deflate.rb'
+    - 'lib/zip_tricks/block_write.rb'
+    - 'lib/zip_tricks/file_reader.rb'
+    - 'lib/zip_tricks/streamer/deflated_writer.rb'
+
+# Offense count: 1
+# Configuration parameters: MinBodyLength.
+Style/GuardClause:
+  Exclude:
+    - 'lib/zip_tricks/file_reader.rb'
+
+# Offense count: 2
+Style/StructInheritance:
+  Exclude:
+    - 'lib/zip_tricks/streamer/entry.rb'
+    - 'spec/zip_tricks/zip_writer_spec.rb'

data/.travis.yml

@@ -9,4 +9,6 @@ cache: bundler
 matrix:
   allow_failures:
     - rvm: jruby-9.0
-script: bundle exec rspec
+script:
+  - bundle exec rubocop -c .rubocop.yml --force-exclusion
+  - bundle exec rspec

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 4.5.0
+
+* Rename `Streamer#add_compressed_entry` and `SizeEstimator#add_compressed_entry` to `add_deflated_entry`
+  to indicate the type of compression that is going to get used.
+* Make  `Streamer#write_(deflated|stored)_file` return a writable object that can be `.close`d, to
+  permit usage of those methods in situations where suspending a block is inconvenient (make deferred writing possible).
+* Fix CRC32 checksums in `Streamer#write_deflated_file`
+* Add `Streamer#update_last_entry_and_write_data_descriptor` to permit externally-driven flows that use data descriptors
+
 ## 4.4.2
 
 * Add 2.4 to Travis rubies

data/Rakefile

@@ -1,12 +1,15 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 require 'yard'
+require 'rubocop/rake_task'
 
 YARD::Rake::YardocTask.new(:doc) do |t|
-  # The dash has to be between the two to "divide" the source files and 
+  # The dash has to be between the two to "divide" the source files and
   # miscellaneous documentation files that contain no code
   t.files = ['lib/**/*.rb', '-', 'LICENSE.txt', 'IMPLEMENTATION_DETAILS.md']
 end
 
 RSpec::Core::RakeTask.new(:spec)
-task :default => :spec
+task default: :spec
+
+RuboCop::RakeTask.new

data/examples/archive_size_estimate.rb

@@ -1,13 +1,15 @@
+# frozen_string_literal: true
+
 require_relative '../lib/zip_tricks'
 
-# Predict how large a ZIP file is going to be without having access to the actual
-# file contents, but using just the filenames (influences the file size) and the size
-# of the files
+# Predict how large a ZIP file is going to be without having access to
+# the actual file contents, but using just the filenames (influences the
+# file size) and the size of the files
 zip_archive_size_in_bytes = ZipTricks::SizeEstimator.estimate do |zip|
   # Pretend we are going to make a ZIP file which contains a few
   # MP4 files (those do not compress all too well)
-  zip.add_stored_entry(filename: "MOV_1234.MP4", size: 898090)
-  zip.add_stored_entry(filename: "MOV_1235.MP4", size: 7855126)
+  zip.add_stored_entry(filename: 'MOV_1234.MP4', size: 898_090)
+  zip.add_stored_entry(filename: 'MOV_1235.MP4', size: 7_855_126)
 end
 
-puts zip_archive_size_in_bytes #=> 8753467
+puts zip_archive_size_in_bytes #=> 8_753_467

data/examples/config.ru

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 require File.dirname(__FILE__) + '/rack_application.rb'
 
 # Demonstrates a Rack app that can offer a ZIP download composed
 # at runtime (see rack_application.rb)
-run ZipDownload.new
+run ZipDownload.new

data/examples/parallel_compression_with_block_deflate.rb

@@ -1,50 +1,58 @@
+# frozen_string_literal: true
+
 require_relative '../lib/zip_tricks'
 require 'tempfile'
 
 # This shows how to perform compression in parallel (a-la pigz, but in a less
-# advanced fashion since the compression tables are not shared - to minimize shared state).
+# advanced fashion since the compression tables are not shared - to
+# minimize shared state).
 #
-# When using this approach, compressing a large file can be performed as a map-reduce operation.
-# First you prepare all the data per part of your (potentially very large) file, and then you use
-# the reduce task to combine that data into one linear zip. In this example we will generate threads
-# and collect their return values in the order the threads were launched, which guarantees a consistent
-# reduce.
+# When using this approach, compressing a large file can be performed as a
+# map-reduce operation.
+# First you prepare all the data per part of your (potentially very large) file,
+# and then you use the reduce task to combine that data into one linear zip.
+# In this example we will generate threads and collect their return values in
+# the order the threads were launched, which guarantees a consistent reduce.
 #
 # So, let each thread generate a part of the file, and also
 # compute the CRC32 of it. The thread will compress it's own part
-# as well, in an independent deflate segment - the threads do not share anything. You could also
-# multiplex this over multiple processes or even machines.
+# as well, in an independent deflate segment - the threads do not share
+# anything. You could also multiplex this over multiple processes or
+# even machines.
 threads = (0..12).map do
   Thread.new do
     source_tempfile = Tempfile.new 't'
     source_tempfile.binmode
-  
+
     # Fill the part with random content
     12.times { source_tempfile << Random.new.bytes(1 * 1024 * 1024) }
     source_tempfile.rewind
-  
+
     # Compute the CRC32 of the source file
     part_crc = ZipTricks::StreamCRC32.from_io(source_tempfile)
     source_tempfile.rewind
-  
+
     # Create a compressed part
     compressed_tempfile = Tempfile.new('tc')
     compressed_tempfile.binmode
-    ZipTricks::BlockDeflate.deflate_in_blocks(source_tempfile, compressed_tempfile)
-    
+    ZipTricks::BlockDeflate.deflate_in_blocks(source_tempfile,
+                                              compressed_tempfile)
+
     source_tempfile.close!
     # The data that the splicing process needs.
     [compressed_tempfile, part_crc, source_tempfile.size]
   end
 end
 
-# Threads return us a tuple with [compressed_tempfile, source_part_size, source_part_crc]
+# Threads return us a tuple with [compressed_tempfile, source_part_size,
+# source_part_crc]
 compressed_tempfiles_and_crc_of_parts = threads.map(&:join).map(&:value)
 
-# Now we need to compute the CRC32 of the _entire_ file, and it has to be the CRC32
-# of the _source_ file (uncompressed), not of the compressed variant. Handily we know
+# Now we need to compute the CRC32 of the _entire_ file, and it has to be
+# the CRC32 of the _source_ file (uncompressed), not of the compressed variant.
+# Handily we know
 entire_file_crc = ZipTricks::StreamCRC32.new
-compressed_tempfiles_and_crc_of_parts.each do | _, source_part_crc, source_part_size|
+compressed_tempfiles_and_crc_of_parts.each do |_, source_part_crc, source_part_size|
   entire_file_crc.append(source_part_crc, source_part_size)
 end
 
@@ -56,7 +64,7 @@ ZipTricks::BlockDeflate.write_terminator(last_compressed_part)
 # To figure that out we just sum the sizes of the files
 compressed_part_files = compressed_tempfiles_and_crc_of_parts.map(&:first)
 size_of_deflated_segment = compressed_part_files.map(&:size).inject(&:+)
-size_of_uncompressed_file = compressed_tempfiles_and_crc_of_parts.map{|e| e[2]}.inject(&:+)
+size_of_uncompressed_file = compressed_tempfiles_and_crc_of_parts.map { |e| e[2] }.inject(&:+)
 
 # And now we can create a ZIP with our compressed file in it's entirety.
 # We use a File as a destination here, but you can also use a socket or a
@@ -64,8 +72,11 @@ size_of_uncompressed_file = compressed_tempfiles_and_crc_of_parts.map{|e| e[2]}.
 # made for streaming.
 output = File.open('zip_created_in_parallel.zip', 'wb')
 
-ZipTricks::Streamer.open(output) do | zip |
-  zip.add_compressed_entry("parallel.bin", size_of_uncompressed_file, entire_file_crc.to_i, size_of_deflated_segment)
+ZipTricks::Streamer.open(output) do |zip|
+  zip.add_deflated_entry('parallel.bin',
+                         size_of_uncompressed_file,
+                         entire_file_crc.to_i,
+                         size_of_deflated_segment)
   compressed_part_files.each do |part_file|
     part_file.rewind
     while blob = part_file.read(2048)

data/examples/rack_application.rb

@@ -1,26 +1,29 @@
+# frozen_string_literal: true
+
 require_relative '../lib/zip_tricks'
 
 # An example of how you can create a Rack endpoint for your ZIP downloads.
 # NEVER run this in production - it is a huge security risk.
 # What this app will do is pick PATH_INFO (your request URL path)
 # and grab a file located at this path on your filesystem. The file will then
-# be added to a ZIP archive created completely programmatically. No data will be cached
-# on disk and the contents of the ZIP file will _not_ be buffered in it's entirety
-# before sending. Unless you use a buffering Rack server of course (WEBrick or Thin).
+# be added to a ZIP archive created completely programmatically. No data will
+# be cached on disk and the contents of the ZIP file will _not_ be buffered in
+# it's entirety before sending. Unless you use a buffering Rack server of
+# course (WEBrick or Thin).
 class ZipDownload
   def call(env)
     file_path = env['PATH_INFO'] # Should be the absolute path on the filesystem
-    
+
     # Open the file for binary reading
     f = File.open(file_path, 'rb')
     filename = File.basename(file_path)
-    
-    # Compute the CRC32 upfront. We do not use local footers for post-computing the CRC32,
-    # so you _do_ have to precompute it beforehand. Ideally, you would do that before
-    # storing the files you will be sending out later on.
+
+    # Compute the CRC32 upfront. We do not use local footers for post-computing
+    # the CRC32, so you _do_ have to precompute it beforehand. Ideally, you
+    # would do that before storing the files you will be sending out later on.
     crc32 = ZipTricks::StreamCRC32.from_io(f)
     f.rewind
-    
+
     # Compute the size of the download, so that a
     # real Content-Length header can be sent. Also, if your download
     # stops at some point, the downloading browser will be able to tell
@@ -30,7 +33,7 @@ class ZipDownload
     size = ZipTricks::SizeEstimator.estimate do |ar|
       ar.add_stored_entry(filename, f.size)
     end
-    
+
     # Create a suitable Rack response body, that will support each(),
     # close() and all the other methods. We can then return it up the stack.
     zip_response_body = ZipTricks::RackBody.new do |zip|
@@ -38,20 +41,25 @@ class ZipDownload
         # We are adding only one file to the ZIP here, but you could do that
         # with an arbitrary number of files of course.
         zip.add_stored_entry(filename: filename, size: f.size, crc32: crc32)
-        # Write the contents of the file. It is stored, so the writes go directly
-        # to the Rack output, bypassing any RubyZip deflaters/compressors. In fact you
-        # are yielding the "blob" string here directly to the Rack server handler.
+        # Write the contents of the file. It is stored, so the writes go
+        # directly to the Rack output, bypassing any RubyZip
+        # deflaters/compressors. In fact you are yielding the "blob" string
+        # here directly to the Rack server handler.
         IO.copy_stream(f, zip)
       ensure
         f.close # Make sure the opened file we read from gets closed
       end
     end
-    
+
     # Add a Content-Disposition so that the download has a .zip extension
     # (this will not work well with UTF-8 filenames on Windows, but hey!)
-    content_disposition = 'attachment; filename=%s.zip' % filename
-    
+    content_disposition = format('attachment; filename=%s.zip', filename)
+
     # and return the response, adding the Content-Length we have computed earlier
-    [200, {'Content-Length' => size.to_s, 'Content-Disposition' => content_disposition}, zip_response_body]
+    [
+      200,
+      {'Content-Length' => size.to_s, 'Content-Disposition' => content_disposition},
+      zip_response_body
+    ]
   end
 end

data/lib/zip_tricks.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
+# Require all the sub-components except myself
 module ZipTricks
-  # Require all the sub-components except myself
-  Dir.glob(__dir__ + '/**/*.rb').sort.each {|p| require p unless p == __FILE__ }
+  Dir.glob(__dir__ + '/**/*.rb').sort.each { |p| require p unless p == __FILE__ }
 end

data/lib/zip_tricks/block_deflate.rb

@@ -1,45 +1,57 @@
+# frozen_string_literal: true
+
 require 'zlib'
 
 # Permits Deflate compression in independent blocks. The workflow is as follows:
 #
-# * Run every block to compress through deflate_chunk, remove the header, footer and adler32 from the result
-# * Write out the compressed block bodies (the ones deflate_chunk returns)to your output, in sequence
+# * Run every block to compress through deflate_chunk, remove the header,
+#   footer and adler32 from the result
+# * Write out the compressed block bodies (the ones deflate_chunk returns)
+#   to your output, in sequence
 # * Write out the footer (\03\00)
 #
-# The resulting stream is guaranteed to be handled properly by all zip unarchiving tools, including the
-# BOMArchiveHelper/ArchiveUtility on OSX.
+# The resulting stream is guaranteed to be handled properly by all zip
+# unarchiving tools, including the BOMArchiveHelper/ArchiveUtility on OSX.
 #
 # You could also build a compressor for Rubyzip using this module quite easily,
 # even though this is outside the scope of the library.
 #
-# When you deflate the chunks separately, you need to write the end marker yourself (using `write_terminator`).
-# If you just want to deflate a large IO's contents, use `deflate_in_blocks_and_terminate` to have the end marker
-# written out for you.
+# When you deflate the chunks separately, you need to write the end marker
+# yourself (using `write_terminator`).
+# If you just want to deflate a large IO's contents, use
+# `deflate_in_blocks_and_terminate` to have the end marker written out for you.
 #
 # Basic usage to compress a file in parts:
-# 
+#
 #     source_file = File.open('12_gigs.bin', 'rb')
 #     compressed = Tempfile.new
-#     # Will not compress everything in memory, but do it per chunk to spare memory. `compressed`
+#     # Will not compress everything in memory, but do it per chunk to spare
+#       memory. `compressed`
 #     # will be written to at the end of each chunk.
-#     ZipTricks::BlockDeflate.deflate_in_blocks_and_terminate(source_file, compressed)
-# 
-# You can also do the same to parts that you will later concatenate together elsewhere, in that case
-# you need to skip the end marker:
-# 
+#     ZipTricks::BlockDeflate.deflate_in_blocks_and_terminate(source_file,
+#                                                             compressed)
+#
+# You can also do the same to parts that you will later concatenate together
+# elsewhere, in that case you need to skip the end marker:
+#
 #     compressed = Tempfile.new
-#     ZipTricks::BlockDeflate.deflate_in_blocks(File.open('part1.bin', 'rb), compressed)
-#     ZipTricks::BlockDeflate.deflate_in_blocks(File.open('part2.bin', 'rb), compressed)
-#     ZipTricks::BlockDeflate.deflate_in_blocks(File.open('partN.bin', 'rb), compressed)
+#     ZipTricks::BlockDeflate.deflate_in_blocks(File.open('part1.bin', 'rb),
+#                                               compressed)
+#     ZipTricks::BlockDeflate.deflate_in_blocks(File.open('part2.bin', 'rb),
+#                                               compressed)
+#     ZipTricks::BlockDeflate.deflate_in_blocks(File.open('partN.bin', 'rb),
+#                                               compressed)
 #     ZipTricks::BlockDeflate.write_terminator(compressed)
-# 
+#
 # You can also elect to just compress strings in memory (to splice them later):
-# 
+#
 #     compressed_string = ZipTricks::BlockDeflate.deflate_chunk(big_string)
-module ZipTricks::BlockDeflate
-  DEFAULT_BLOCKSIZE = 1024*1024*5
-  END_MARKER = [3, 0].pack("C*")
-  VALID_COMPRESSIONS = (Zlib::DEFAULT_COMPRESSION..Zlib::BEST_COMPRESSION).to_a.freeze # Zlib::NO_COMPRESSION..
+
+class ZipTricks::BlockDeflate
+  DEFAULT_BLOCKSIZE = 1_024 * 1024 * 5
+  END_MARKER = [3, 0].pack('C*')
+  # Zlib::NO_COMPRESSION..
+  VALID_COMPRESSIONS = (Zlib::DEFAULT_COMPRESSION..Zlib::BEST_COMPRESSION).to_a.freeze
   # Write the end marker (\x3\x0) to the given IO.
   #
   # `output_io` can also be a {ZipTricks::Streamer} to expedite ops.
@@ -83,7 +95,10 @@ module ZipTricks::BlockDeflate
   # @param level [Fixnum] Zlib compression level (defaults to `Zlib::DEFAULT_COMPRESSION`)
   # @param block_size [Fixnum] The block size to use (defaults to `DEFAULT_BLOCKSIZE`)
   # @return [Fixnum] number of bytes written to `output_io`
-  def self.deflate_in_blocks_and_terminate(input_io, output_io, level: Zlib::DEFAULT_COMPRESSION, block_size: DEFAULT_BLOCKSIZE)
+  def self.deflate_in_blocks_and_terminate(input_io,
+                                           output_io,
+                                           level: Zlib::DEFAULT_COMPRESSION,
+                                           block_size: DEFAULT_BLOCKSIZE)
     bytes_written = deflate_in_blocks(input_io, output_io, level: level, block_size: block_size)
     bytes_written + write_terminator(output_io)
   end
@@ -100,7 +115,10 @@ module ZipTricks::BlockDeflate
   # @param level [Fixnum] Zlib compression level (defaults to `Zlib::DEFAULT_COMPRESSION`)
   # @param block_size [Fixnum] The block size to use (defaults to `DEFAULT_BLOCKSIZE`)
   # @return [Fixnum] number of bytes written to `output_io`
-  def self.deflate_in_blocks(input_io, output_io, level: Zlib::DEFAULT_COMPRESSION, block_size: DEFAULT_BLOCKSIZE)
+  def self.deflate_in_blocks(input_io,
+                             output_io,
+                             level: Zlib::DEFAULT_COMPRESSION,
+                             block_size: DEFAULT_BLOCKSIZE)
     bytes_written = 0
     while block = input_io.read(block_size)
       deflated = deflate_chunk(block, level: level)