ruby-xz 0.2.3 → 1.0.0

data/lib/xz/stream_writer.rb

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
 #--
-# (The MIT license)
-#
 # Basic liblzma-bindings for Ruby.
 #
-# Copyright © 2012, 2015 Marvin Gülker
+# Copyright © 2011-2018 Marvin Gülker et al.
+#
+# See AUTHORS for the full list of contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the ‘Software’),
@@ -33,10 +33,10 @@
 # XZ-compressed data to. Here’s an ASCII art image to demonstrate way
 # data flows when using StreamWriter to write to a compressed file:
 #
-#           +----------------+  +------------+
-#   YOUR  =>|StreamWriter's  |=>|Wrapped IO's|=> ACTUAL
-#   DATA  =>|internal buffers|=>|buffers     |=>  FILE
-#           +----------------+  +------------+
+#           +-----------------+  +------------+
+#   YOUR  =>|StreamWriter's   |=>|Wrapped IO's|=> ACTUAL
+#   DATA  =>|(liblzma) buffers|=>|buffers     |=>  FILE
+#           +-----------------+  +------------+
 #
 # This graphic also illustrates why it is unlikely to see written data
 # directly appear in the file on your harddisk; the data is cached at
@@ -46,361 +46,220 @@
 # instance and then the wrapped IO object (in *exactly* that order,
 # otherwise data loss and unexpected exceptions may occur!).
 #
-# As it might be tedious to always remember the correct closing order,
-# it’s possible to pass a filename to the ::open method. In this case,
-# StreamWriter will open the file internally and also takes care
-# closing it when you call the #close method.
-#
-# *WARNING*: The closing behaviour described above is subject to
-# change in the next major version. In the future, wrapped IO
-# objects are automatically closed always, regardless of whether you
-# passed a filename or an IO instance. This is to sync the API with
-# Ruby’s own Zlib::GzipWriter. To retain the old behaviour, call
-# the #finish method (which is also in sync with the Zlib API).
-#
-# See the +io-like+ gem’s documentation for the IO-writing methods
-# available for this class (although you’re probably familiar with
-# them through Ruby’s own IO class ;-)).
-#
-# == Example
-#
-# Together with the <tt>archive-tar-minitar</tt> gem, this library
-# can be used to create XZ-compressed TAR archives (these commonly
-# use a file extension of <tt>.tar.xz</tt> or rarely <tt>.txz</tt>).
+# Calling the #close method closes both the XZ writer and the
+# underlying IO object in the correct order. This is akin to the
+# behaviour exposed by Ruby's own Zlib::GzipWriter class. If you
+# expressly don't want to close the underlying IO instance, you need
+# to manually call StreamWriter#finish and never call
+# StreamWriter#close. Instead, you then close your IO object manually
+# using IO#close once you're done with it.
 #
-#   XZ::StreamWriter.open("foo.tar.xz") do |txz|
-#     # This automatically closes txz
-#     Archive::Tar::Minitar.pack("foo", txz)
-#   end
+# *NOTE*: Using #finish inside the +open+ method's block allows
+# you to continue using that writer's File instance as it is
+# returned by #finish.
 class XZ::StreamWriter < XZ::Stream
 
+  # Compression level used for this writer (set on instanciation).
+  attr_reader :level
+  # Checksum algorithm in use.
+  attr_reader :check
+
   # call-seq:
-  #  new(delegate, compression_level = 6, opts = {}) → writer
-  #  new(delegate, compression_level = 6, opts = {}){|writer| …} → obj
+  #   open(filename [, compression_level = 6 [, options ]]) → stream_writer
+  #   open(filename [, compression_level = 6 [, options ]]){|sw| ...} → stream_writer
   #
-  # Creates a new StreamWriter instance. The block form automatically
-  # calls the #close method when the block has finished executing.
+  # Creates a new instance for writing to a compressed file. The File
+  # instance is opened internally and then wrapped via ::new. The
+  # block form automatically closes both the liblzma stream and the
+  # internal File instance in the correct order. The non-block form
+  # does neither, leaving it to you to call #finish or #close later.
   #
   # === Parameters
-  # [delegate]
-  #   An IO object to write the data to
-  #
-  # [compression_level (6)]
-  #   Compression strength. Higher values indicate a smaller result,
-  #   but longer compression time. Maximum is 9.
-  #
-  # [opts]
-  #   Options hash. Possible values are (defaults indicated in
-  #   parantheses):
-  #
-  #   [:check (:crc64)]
-  #     The checksum algorithm to use for verifying
-  #     the data inside the archive. Possible values are:
-  #     * :none
-  #     * :crc32
-  #     * :crc64
-  #     * :sha256
-  #
-  #   [:extreme (false)]
-  #     Tries to get the last bit out of the compression.
-  #     This may succeed, but you can end up with *very*
-  #     long computation times.
+  # [filename]
+  #   The file to open.
+  # [sw (block argument)]
+  #   The created StreamWriter instance.
   #
-  # [writer]
-  #   Block argument. self of the new instance.
+  # See ::new for the other parameters.
   #
   # === Return value
+  # Returns the newly created instance.
   #
-  # The block form returns the block’s last expression, the nonblock
-  # form returns the newly created instance.
-  #
-  # === Deprecations
-  #
-  # The old API for this method as it was documented in version 0.2.1
-  # still works, but is deprecated. Please change to the new API as
-  # soon as possible.
-  #
-  # *WARNING*: The closing behaviour of the block form is subject to
-  # upcoming change. In the next major release the wrapped IO *will*
-  # be automatically closed, unless you call #finish to prevent that.
+  # === Remarks
+  # Starting with version 1.0.0, the block form also returns the newly
+  # created instance rather than the block's return value. This is
+  # in line with Ruby's own GzipWriter.open API.
   #
   # === Example
-  #
-  #   # Wrap it around a file
-  #   f = File.open("data.xz")
-  #   w = XZ::StreamWriter.new(f)
-  #
-  #   # Use SHA256 as the checksum and use a higher compression level
-  #   # than the default (6)
-  #   f = File.open("data.xz")
-  #   w = XZ::StreamWriter.new(f, 8, :check => :sha256)
-  #
-  #   # Instruct liblzma to use ultra-really-high compression
-  #   # (may take eternity)
-  #   f = File.open("data.xz")
-  #   w = XZ::StreamWriter.new(f, 9, :extreme => true)
-  def initialize(delegate, compression_level = 6, *args, &block)
-    if delegate.respond_to?(:to_io)
-      # Correct use with IO
-      super(delegate.to_io)
-      @autoclose = false
-    else
-      # Deprecated use of filename
-      XZ.deprecate "Calling XZ::StreamWriter.new with a filename is deprecated, use XZ::StreamWriter.open instead"
-
-      @autoclose = true
-      super(File.open(delegate, "wb"))
-    end
-
-    # Flag for #finish method
-    @finish = false
-
-    opts = {}
-    if args[0].kind_of?(Hash) # New API
-      opts = args[0]
-      opts[:check] ||= :crc64
-      opts[:extreme] ||= false
-    else # Old API
-      # no arguments may also happen in new API
-      unless args.empty?
-        XZ.deprecate "Calling XZ::StreamWriter withm ore than 2 explicit arguments is deprecated, use options hash instead."
-      end
-
-      opts[:check] = args[0] || :crc64
-      opts[:extreme] = args[1] || false
-    end
-
-    # TODO: Check argument validity...
-
-    # Initialize the internal LZMA stream for encoding
-    res = XZ::LibLZMA.lzma_easy_encoder(@lzma_stream.pointer,
-                                  compression_level | (opts[:extreme] ? XZ::LibLZMA::LZMA_PRESET_EXTREME : 0),
-                                  XZ::LibLZMA::LZMA_CHECK[:"lzma_check_#{opts[:check]}"])
-    XZ::LZMAError.raise_if_necessary(res)
+  #     # Normal usage
+  #     XZ::StreamWriter.open("myfile.txt.xz") do |xz|
+  #       xz.puts "Compress this line"
+  #       xz.puts "And this line as well"
+  #     end
+  #
+  #     # If for whatever reason you want to do something else with
+  #     # the internally opened file:
+  #     file = nil
+  #     XZ::StreamWriter.open("myfile.txt.xz") do |xz|
+  #       xz.puts "Compress this line"
+  #       xz.puts "And this line as well"
+  #       file = xz.finish
+  #     end
+  #     # At this point, the liblzma stream has been closed, but `file'
+  #     # now contains the internally created File instance, which is
+  #     # still open. Don't forget to close it yourself at some point
+  #     # to flush it.
+  #     file.close
+  #
+  #     # Or just don't use the block form:
+  #     xz = StreamWriter.open("myfile.txt.xz")
+  #     xz.puts "Compress this line"
+  #     xz.puts "And this line as well"
+  #     file = xz.finish
+  #     file.close # Don't forget to close it manually (or use xz.close instead of xz.finish above)
+  def self.open(filename, **args)
+    file = File.open(filename, "wb")
+    writer = new(file, **args)
 
     if block_given?
       begin
-        yield(self)
+        yield(writer)
       ensure
-        close unless closed?
+        # Close both writer and delegate IO via writer.close
+        # unless the writer has manually been finished (usually
+        # not closing the delegate IO then).
+        writer.close unless writer.finished?
       end
     end
+
+    writer
   end
 
-  # call-seq:
-  #   open(filename, compression_level = 6, opts = {}) → writer
-  #   open(filename, compression_level = 6, opts = {}){|writer| …} → obj
-  #
-  # Opens a file from disk and wraps an XZ::StreamWriter instance
-  # around the resulting file IO object. This is a convenience method
-  # mostly equivalent to
-  #
-  #   file = File.open(filename, "wb")
-  #   writer = XZ::StreamWriter.new(file, compression_level, opts)
-  #
-  # , except that you don’t have to explicitely close the File
-  # instance, this is done automatically for you when you call #close.
-  # Beware the Deprecations section in this regard.
+  # Creates a new instance that is wrapped around the given IO instance.
   #
   # === Parameters
+  # ==== Positional parameters
+  # [delegate_io]
+  #   The IO instance to wrap. It has to be opened in binary mode,
+  #   otherwise the data it writes to the hard disk will be corrupt.
   #
-  # [filename]
-  #   Path to a file on the disk to open. This file should exist and be
-  #   writable, otherwise you may get Errno exceptions.
-  #
-  # [opts]
-  #   Options hash. See ::new for a description of the possible
-  #   options.
-  #
-  # [writer]
-  #   Block argument. self of the new instance.
+  # ==== Keyword arguments
+  # [compression_level (6)]
+  #   Compression strength. Higher values indicate a
+  #   smaller result, but longer compression time. Maximum
+  #   is 9.
+  # [:check (:crc64)]
+  #   The checksum algorithm to use for verifying
+  #   the data inside the archive. Possible values are:
+  #   * :none
+  #   * :crc32
+  #   * :crc64
+  #   * :sha256
+  # [:extreme (false)]
+  #   Tries to get the last bit out of the
+  #   compression. This may succeed, but you can end
+  #   up with *very* long computation times.
+  # [:external_encoding (Encoding.default_external)]
+  #   Transcode to this encoding when writing. Defaults
+  #   to Encoding.default_external, which by default is
+  #   set from the environment.
   #
   # === Return value
+  # Returns the newly created instance.
   #
-  # The block form returns the blocks last expression, the nonblock
-  # form returns the newly created XZ::StreamWriter instance.
-  #
-  # === Deprecations
-  #
-  # In the API up to and including version 0.2.1 this method was an
-  # alias for ::new. This continues to work for now, but using it
-  # as an alias for ::new is deprecated. The next major version will
-  # only accept a string as a parameter for this method.
+  # === Remarks
+  # This method does not close the underlying IO nor does it automatically
+  # flush libzlma. You'll need to do that manually using #close or #finish.
+  # See ::open for a method that supports a block with auto-closing.
   #
-  # *WARNING*: Future versions of ruby-xz will always close the
-  # wrapped IO, regardless of whether you pass in your own IO or use
-  # this convenience method, unless you call #finish to prevent that.
+  # This method used to accept a block in earlier versions. This
+  # behaviour has been removed in version 1.0.0 to synchronise the API
+  # with Ruby's own GzipWriter.new.
   #
   # === Example
-  #
-  #   w = XZ::StreamWriter.new("compressed_data.xz")
-  def self.open(filename, compression_level = 6, *args, &block)
-    if filename.respond_to?(:to_io)
-      # Deprecated use of IO
-      XZ.deprecate "Calling XZ::StreamWriter.open with an IO is deprecated, use XZ::StreamReader.new instead."
-      new(filename.to_io, compression_level, *args, &block)
-    else
-      # Correct use with filename
-      file = File.open(filename, "wb")
-
-      obj = new(file, compression_level, *args)
-      obj.instance_variable_set(:@autoclose, true) # Only needed during deprecation phase, see #close
-
-      if block_given?
-        begin
-          block.call(obj)
-        ensure
-          obj.close unless obj.closed?
-        end
-      else
-        obj
-      end
-    end
+  #     # Normal usage:
+  #     file = File.open("myfile.txt.xz", "wb") # Note binary mode
+  #     xz = XZ::StreamWriter.new(file)
+  #     xz.puts("Compress this line")
+  #     xz.puts("And this second line")
+  #     xz.close # Closes both the libzlma stream and `file'
+  #
+  #     # Expressly closing the delegate IO manually:
+  #     File.open("myfile.txt.xz", "wb") do |file| # Note binary mode
+  #       xz = XZ::StreamWriter.new(file)
+  #       xz.puts("Compress this line")
+  #       xz.puts("And this second line")
+  #       xz.finish # Flushes libzlma, but keeps `file' open.
+  #     end # Here, `file' is closed.
+  def initialize(delegate_io, level: 6, check: :crc64, extreme: false, external_encoding: nil)
+    super(delegate_io)
+
+    raise(ArgumentError, "Invalid compression level!")  unless (0..9).include?(level)
+    raise(ArgumentError, "Invalid checksum specified!") unless [:none, :crc32, :crc64, :sha256].include?(check)
+
+    set_encoding(external_encoding) if external_encoding
+
+    @check  = check
+    @level  = level
+    @level |= LibLZMA::LZMA_PRESET_EXTREME if extreme
+
+    res = XZ::LibLZMA.lzma_easy_encoder(@lzma_stream.to_ptr,
+                                    @level,
+                                    XZ::LibLZMA.const_get(:"LZMA_CHECK_#{@check.upcase}"))
+    XZ::LZMAError.raise_if_necessary(res)
   end
 
-  # Closes this StreamWriter instance and flushes all internal buffers.
-  # Don’t use it afterwards anymore.
-  #
-  # === Return value
-  #
-  # The total number of bytes written, i.e. the size of the compressed
-  # data.
-  #
-  # === Example
-  #
-  #   w.close #=> 424
-  #
-  # === Remarks
-  #
-  # If you passed an IO object to ::new, this method doesn’t close it,
-  # you have to do that yourself.
-  #
-  # *WARNING*: The next major release will change this behaviour.
-  # In the future, the wrapped IO object will always be closed.
-  # Use the #finish method for keeping it open.
-  def close
-    super
-
-    #1. Close the current block ("file") (an XZ stream may actually include
-    #   multiple compressed files, which however is not supported by
-    #   this library). For this we have to tell liblzma that
-    #   the next bytes we pass to it are the last bytes (by means of
-    #   the FINISH action). Just that we don’t pass any new input ;-)
-
-    output_buffer_p         = FFI::MemoryPointer.new(XZ::CHUNK_SIZE)
+  # Mostly like IO#write. Additionally it raises an IOError
+  # if #finish has been called previously.
+  def write(*args)
+    raise(IOError, "Cannot write to a finished liblzma stream") if @finished
 
-    # Get any pending data (LZMA_FINISH causes libzlma to flush its
-    # internal buffers) and write it out to our wrapped IO.
-    loop do
-      @lzma_stream[:next_out]  = output_buffer_p
-      @lzma_stream[:avail_out] = output_buffer_p.size
+    origpos = @pos
 
-      res = XZ::LibLZMA.lzma_code(@lzma_stream.pointer, XZ::LibLZMA::LZMA_ACTION[:lzma_finish])
-      XZ::LZMAError.raise_if_necessary(res)
-
-      @delegate_io.write(output_buffer_p.read_string(output_buffer_p.size - @lzma_stream[:avail_out]))
-
-      break unless @lzma_stream[:avail_out] == 0
-    end
-
-    # 2. Close the whole XZ stream.
-    res = XZ::LibLZMA.lzma_end(@lzma_stream.pointer)
-    XZ::LZMAError.raise_if_necessary(res)
+    args.each do |arg|
+      @pos += arg.to_s.bytesize
 
-    unless @finish
-      # New API: Close the wrapped IO
-      #@delegate_io.close
+      # Apply external encoding if requested
+      if @external_encoding && @external_encoding != Encoding::BINARY
+        arg = arg.to_s.encode(@external_encoding)
+      end
 
-      # Old API:
-      # 2b. If we wrapped a file automatically, close it.
-      if @autoclose
-        @delegate_io.close
-      else
-        XZ.deprecate "XZ::StreamWriter#close will automatically close the wrapped IO in the future. Use #finish to prevent that."
+      lzma_code(arg.to_s, XZ::LibLZMA::LZMA_RUN) do |compressed|
+        @delegate_io.write(compressed)
       end
     end
 
-    # 3. Return the number of bytes written in total.
-    @lzma_stream[:total_out]
+    @pos - origpos # Return number of bytes consumed from input
   end
 
-  # If called in the block form of ::new or ::open, prevents the
-  # wrapped IO from being closed, only the LZMA stream is closed
-  # then. If called outside the block form of ::new and open, behaves
-  # like #close, but only closes the underlying LZMA stream. The
-  # wrapped IO object is kept open.
-  #
-  # === Return value
-  #
-  # Returns the wrapped IO object. This allows you to wire the File
-  # instance out of a StreamReader instance that was created with
-  # ::open.
-  #
-  # === Example
-  #
-  #   # Nonblock form
-  #   f = File.open("foo.xz", "wb")
-  #   w = XZ::StreamReader.new(f)
-  #   # ...
-  #   w.finish
-  #   # f is still open here!
-  #
-  #   # Block form
-  #   f = XZ::StreamReader.open("foo.xz") do |w|
-  #     # ...
-  #     w.finish
-  #   end
-  #   # f now is an *open* File instance of mode "wb".
+  # Like superclass' method, but also ensures liblzma flushes all
+  # compressed data to the delegate IO.
   def finish
-    # Do not close wrapped IO object in #close
-    @finish = true
-    close
-
-    @delegate_io
+    lzma_code("", XZ::LibLZMA::LZMA_FINISH) { |compressed| @delegate_io.write(compressed) }
+    super
   end
 
-  # call-seq:
-  #   pos()  → an_integer
-  #   tell() → an_integer
+  # Abort the current compression process and reset everything
+  # to the start. Writing into this writer will cause existing data
+  # on the underlying IO to be overwritten after this method has been
+  # called.
   #
-  # Total number of input bytes read so far from what you supplied to
-  # any writer method.
-  def pos
-    @lzma_stream[:total_in]
-  end
-  alias tell pos
-
-  private
-
-  # Called by io-like’s write methods such as #write. Does the heavy
-  # work of feeding liblzma the uncompressed data and reading the
-  # returned compressed data.
-  def unbuffered_write(data)
-    output_buffer_p = FFI::MemoryPointer.new(XZ::CHUNK_SIZE)
-    input_buffer_p  = FFI::MemoryPointer.from_string(data) # This adds a terminating NUL byte we don’t want to compress!
-
-    @lzma_stream[:next_in]  = input_buffer_p
-    @lzma_stream[:avail_in] = input_buffer_p.size - 1 # Don’t hand the terminating NUL
-
-    loop do
-      @lzma_stream[:next_out]  = output_buffer_p
-      @lzma_stream[:avail_out] = output_buffer_p.size
-
-      # Compress the data
-      res = XZ::LibLZMA.lzma_code(@lzma_stream.pointer, XZ::LibLZMA::LZMA_ACTION[:lzma_run])
-      XZ::LZMAError.raise_if_necessary(res) # TODO: Warnings
+  # The delegte IO has to support the #rewind method. Otherwise like
+  # IO#rewind.
+  def rewind
+    super
 
-      # Write the compressed data
-      result = output_buffer_p.read_string(output_buffer_p.size - @lzma_stream[:avail_out])
-      @delegate_io.write(result)
+    res = XZ::LibLZMA.lzma_easy_encoder(@lzma_stream.to_ptr,
+                                    @level,
+                                    XZ::LibLZMA.const_get(:"LZMA_CHECK_#{@check.upcase}"))
+    XZ::LZMAError.raise_if_necessary(res)
 
-      # Loop until liblzma ate the whole data.
-      break if @lzma_stream[:avail_in] == 0
-    end
+    0 # Mimic IO#rewind's return value
+  end
 
-    binary_size(data)
-  rescue XZ::LZMAError => e
-    raise(SystemCallError, e.message)
+  # Human-readable description
+  def inspect
+    "<#{self.class} pos=#{@pos} finished=#{@finished} closed=#{closed?} io=#{@delegate_io.inspect}>"
   end
 
 end