ruby-xz 0.2.1 → 1.0.1

data/lib/xz/stream_reader.rb

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
-# (The MIT license)
-#
+#--
 # Basic liblzma-bindings for Ruby.
 #
-# Copyright © 2012 Marvin Gülker
+# Copyright © 2011-2018 Marvin Gülker et al.
+#
+# See AUTHORS for the full list of contributors.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the ‘Software’),
@@ -22,264 +23,290 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
+#++
 
-#An IO-like reader class for XZ-compressed data, allowing you to
-#access XZ-compressed data as if it was a normal IO object, but
-#please note you can’t seek in the data--this doesn’t make much
-#sense anyway. Where would you want to seek? The plain or the XZ
-#data?
-#
-#A StreamReader object actually wraps another IO object it reads
-#the compressed data from; you can either pass this IO object directly
-#to the ::new method, effectively allowing you to pass any IO-like thing
-#you can imagine (just ensure it is readable), or you can pass a path
-#to a filename to ::new, in which case StreamReader takes care of both
-#opening and closing the file correctly. You can even take it one step
-#further and use the block form of ::new which will automatically call
-#the #close method for you after the block finished. However, if you pass
-#an IO, remember you have to close:
-#
-#1. The StreamReader instance.
-#2. The IO object you passed to ::new.
-#
-#Do it <b>in exactly that order</b>, otherwise you may lose data.
+# An IO-like reader class for XZ-compressed data, allowing you to
+# access XZ-compressed data as if it was a normal IO object, but
+# please note you can’t seek in the data--this doesn’t make much
+# sense anyway. Where would you want to seek? The plain or the XZ
+# data?
 #
-#See the +io-like+ gem’s documentation for the IO-reading methods
-#available for this class (although you’re probably familiar with
-#them through Ruby’s own IO class ;-)).
-#
-#==Example
-#In this example, we’re going to use ruby-xz together with the
-#+archive-tar-minitar+ gem that allows to read tarballs. Used
-#together, the two libraries allow us to read XZ-compressed tarballs.
-#
-#  require "xz"
-#  require "archive/tar/minitar"
-#
-#  XZ::StreamReader.open("foo.tar.xz") do |txz|
-#    # This automatically closes txz
-#    Archive::Tar::Minitar.unpack(txz, "foo")
-#  end
+# A StreamReader object actually wraps another IO object it reads
+# the compressed data from; you can either pass this IO object directly
+# to the ::new method, effectively allowing you to pass any IO-like thing
+# you can imagine (just ensure it is readable), or you can pass a path
+# to a file to ::open, in which case StreamReader will open the path
+# using Ruby's File class internally. If you use ::open's block form,
+# the method will take care of properly closing both the liblzma
+# stream and the File instance correctly.
 class XZ::StreamReader < XZ::Stream
 
-  #The memory limit you set for this reader (in ::new).
+  # The memory limit configured for this lzma decoder.
   attr_reader :memory_limit
-  #The flags you set for this reader (in ::new).
-  attr_reader :flags
 
-  #call-seq:
-  #  new(delegate, memory_limit = XZ::LibLZMA::UINT64_MAX, flags = [:tell_unsupported_check])  → a_stream_reader
-  #  open(delegate, memory_limit = XZ::LibLZMA::UINT64_MAX, flags = [:tell_unsupported_check]) → a_stream_reader
+  # call-seq:
+  #   open(filename [, kw]) → stream_reader
+  #   open(filename [, kw]){|sr| ...} → stream_reader
   #
-  #Creates a new StreamReader instance. If you pass an IO,
-  #remember you have to close *both* the resulting instance
-  #(via the #close method) and the IO object you pass to flush
-  #any internal buffers in order to be able to read all decompressed
-  #data.
-  #==Parameters
-  #[delegate] An IO object to read the data from, or a path
-  #           to a file to open. If you’re in an urgent need to
-  #           pass a plain string, use StringIO from Ruby’s
-  #           standard library. If this is an IO, it must be
-  #           opened for reading.
-  #The other parameters are identical to what the XZ::decompress_stream
-  #method expects.
-  #==Return value
-  #The newly created instance.
-  #==Example
-  #  # Wrap it around a file
-  #  f = File.open("foo.xz")
-  #  r = XZ::StreamReader.new(f)
+  # Open the given file and wrap a new instance around it with ::new.
+  # If you use the block form, both the internally created File instance
+  # and the liblzma stream will be closed automatically for you.
   #
-  #  # Ignore any XZ checksums (may result in invalid data being read!)
-  #  File.open("foo.xz") do |f|
-  #    r = XZ::StreamReader.new(f, XZ::LibLZMA::UINT64_MAX, [:tell_no_check]
-  #  end
+  # === Parameters
+  # [filename]
+  #   Path to the file to open.
+  # [sr (block argument)]
+  #   The created StreamReader instance.
   #
-  #  # Let StreamReader handle file closing automatically
-  #  XZ::StreamReader.new("myfile.xz"){|r| r.raed}
-  def initialize(delegate, memory_limit = XZ::LibLZMA::UINT64_MAX, flags = [:tell_unsupported_check])
-    raise(ArgumentError, "Invalid memory limit set!") unless (0..XZ::LibLZMA::UINT64_MAX).include?(memory_limit)
-    flags.each do |flag|
-      raise(ArgumentError, "Unknown flag #{flag}!") unless [:tell_no_check, :tell_unsupported_check, :tell_any_check, :concatenated].include?(flag)
-    end
-
-    if delegate.respond_to?(:to_io)
-      super(delegate)
-    else
-      @file = File.open(delegate, "rb")
-      super(@file)
-    end
-
-    @memory_limit = memory_limit
-    @flags        = flags
-
-    res = XZ::LibLZMA.lzma_stream_decoder(@lzma_stream,
-                                          @memory_limit,
-                                          @flags.inject(0){|val, flag| val | XZ::LibLZMA.const_get(:"LZMA_#{flag.to_s.upcase}")})
-    XZ::LZMAError.raise_if_necessary(res)
-
-    @input_buffer_p = FFI::MemoryPointer.new(XZ::CHUNK_SIZE)
-
-    # These two are only used in #unbuffered read.
-    @__lzma_finished = false
-    @__lzma_action   = nil
+  # See ::new for a description of the keyword parameters.
+  #
+  # === Return value
+  # The newly created instance.
+  #
+  # === Remarks
+  # Starting with version 1.0.0, the block form also returns the newly
+  # created instance rather than the block's return value. This is
+  # in line with Ruby's own GzipReader.open API.
+  #
+  # === Example
+  #     # Normal usage
+  #     XZ::StreamReader.open("myfile.txt.xz") do |xz|
+  #       puts xz.read #=> I love Ruby
+  #     end
+  #
+  #     # If you really need the File instance created internally:
+  #     file = nil
+  #     XZ::StreamReader.open("myfile.txt.xz") do |xz|
+  #       puts xz.read #=> I love Ruby
+  #       file = xz.finish # prevents closing
+  #     end
+  #     file.close # Now close it manually
+  #
+  #     # Or just don't use the block form:
+  #     xz = XZ::StreamReader.open("myfile.txt.xz")
+  #     puts xz.read #=> I love Ruby
+  #     file = xz.finish
+  #     file.close # Don't forget to close it manually (or use xz.close instead of xz.finish above).
+  def self.open(filename, **args)
+    file = File.open(filename, "rb")
+    reader = new(file, **args)
 
     if block_given?
       begin
-        yield(self)
+        yield(reader)
       ensure
-        close unless closed?
+        # Close both delegate IO and reader.
+        reader.close unless reader.finished?
       end
     end
+
+    reader
   end
-  self.class.send(:alias_method, :open, :new)
 
-  #Closes this StreamReader instance. Don’t use it afterwards
-  #anymore.
-  #==Return value
-  #The total number of bytes decompressed.
-  #==Example
-  #  r.close #=> 6468
-  #==Remarks
-  #If you passed an IO to ::new, this method doesn’t close it, so
-  #you have to close it yourself.
-  def close
-    super
+  # Creates a new instance that is wrapped around the given IO object.
+  #
+  # === Parameters
+  # ==== Positional parameters
+  # [delegate_io]
+  #   The underlying IO object to read the compressed data from.
+  #   This IO object has to have been opened in binary mode,
+  #   otherwise you are likely to receive exceptions indicating
+  #   that the compressed data is corrupt.
+  #
+  # ==== Keyword arguments
+  # [memory_limit (+UINT64_MAX+)]
+  #   If not XZ::LibLZMA::UINT64_MAX, makes liblzma
+  #   use no more memory than +memory_limit+ bytes.
+  # [flags (<tt>[:tell_unsupported_check]</tt>)]
+  #   Additional flags passed to liblzma (an array).
+  #   Possible flags are:
+  #
+  #   [:tell_no_check]
+  #     Spit out a warning if the archive hasn't an
+  #     integrity checksum.
+  #   [:tell_unsupported_check]
+  #     Spit out a warning if the archive
+  #     has an unsupported checksum type.
+  #   [:concatenated]
+  #     Decompress concatenated archives.
+  # [external_encoding (Encoding.default_external)]
+  #   Assume the decompressed data inside the XZ is encoded in
+  #   this encoding. Defaults to Encoding.default_external,
+  #   which in turn defaults to the environment.
+  # [internal_encoding (Encoding.default_internal)]
+  #   Request that the data found in the XZ file (which is assumed
+  #   to be in the encoding specified by +external_encoding+) to
+  #   be transcoded into this encoding. Defaults to Encoding.default_internal,
+  #   which defaults to nil, which means to not transcode anything.
+  #
+  # === Return value
+  # The newly created instance.
+  #
+  # === Remarks
+  # The strings returned from the reader will be in the encoding specified
+  # by the +internal_encoding+ parameter. If that parameter is nil (default),
+  # then they will be in the encoding specified by +external_encoding+.
+  #
+  # This method used to accept a block in earlier versions. Since version 1.0.0,
+  # this behaviour has been removed to synchronise the API with Ruby's own
+  # GzipReader.open.
+  #
+  # This method doesn't close the underlying IO or the liblzma stream.
+  # You need to call #finish or #close manually; see ::open for a method
+  # that takes a block to automate this.
+  #
+  # === Example
+  #     file = File.open("compressed.txt.xz", "rb") # Note binary mode
+  #     xz = XZ::StreamReader.open(file)
+  #     puts xz.read #=> I love Ruby
+  #     xz.close # closes both `xz' and `file'
+  #
+  #     file = File.open("compressed.txt.xz", "rb") # Note binary mode
+  #     xz = XZ::StreamReader.open(file)
+  #     puts xz.read #=> I love Ruby
+  #     xz.finish # closes only `xz'
+  #     file.close # Now close `file' manually
+  def initialize(delegate_io, memory_limit: XZ::LibLZMA::UINT64_MAX, flags: [:tell_unsupported_check], external_encoding: nil, internal_encoding: nil)
+    super(delegate_io)
+    raise(ArgumentError, "When specifying the internal encoding, the external encoding must also be specified") if internal_encoding && !external_encoding
+    raise(ArgumentError, "Memory limit out of range") unless memory_limit > 0 && memory_limit <= XZ::LibLZMA::UINT64_MAX
 
-    # Close the XZ stream
-    res = XZ::LibLZMA.lzma_end(@lzma_stream.pointer)
-    XZ::LZMAError.raise_if_necessary(res)
+    @memory_limit = memory_limit
+    @readbuf = String.new
+    @readbuf.force_encoding(Encoding::BINARY)
 
-    #If we created a File object, close this as well.
-    @file.close if @file
+    if external_encoding
+      encargs = []
+      encargs << external_encoding
+      encargs << internal_encoding if internal_encoding
+      set_encoding(*encargs)
+    end
+
+    @allflags = flags.reduce(0) do |val, flag|
+      flag = XZ::LibLZMA::LZMA_DECODE_FLAGS[flag] || raise(ArgumentError, "Unknown flag #{flag}")
+      val | flag
+    end
 
-    # Return the number of bytes written in total.
-    @lzma_stream[:total_out]
+    res = XZ::LibLZMA.lzma_stream_decoder(@lzma_stream.to_ptr,
+                                      @memory_limit,
+                                      @allflags)
+    XZ::LZMAError.raise_if_necessary(res)
   end
 
-  #call-seq:
-  #  pos()  → an_integer
-  #  tell() → an_integer
+  # Mostly like IO#read. The +length+ parameter refers to the amount
+  # of decompressed bytes to read, not the amount of bytes to read
+  # from the compressed data. That is, if you request a read of 50
+  # bytes, you will receive a string with a maximum length of 50
+  # bytes, regardless of how many bytes this was in compressed form.
   #
-  #Total number of output bytes provided to you yet.
-  def pos
-    @lzma_stream[:total_out]
-  end
-  alias tell pos
+  # Return values are as per IO#read.
+  def read(length = nil, outbuf = String.new)
+    return "".force_encoding(Encoding::BINARY) if length == 0 # Shortcut; retval as per IO#read.
 
-  #Instrcuts liblzma to immediately stop decompression,
-  #rewinds the wrapped IO object and reinitalizes the
-  #StreamReader instance with the same values passed
-  #originally to the ::new method. The wrapped IO object
-  #must support the +rewind+ method for this method to
-  #work; if it doesn’t, this method throws an IOError.
-  #After the exception was thrown, the StreamReader instance
-  #is in an unusable state. You cannot continue using it
-  #(don’t call #close on it either); close the wrapped IO
-  #stream and create another instance of this class.
-  #==Raises
-  #[IOError] The wrapped IO doesn’t support rewinding.
-  #          Do not use the StreamReader instance anymore
-  #          after receiving this exception.
-  #==Remarks
-  #I don’t really like this method, it uses several dirty
-  #tricks to circumvent both io-like’s and liblzma’s control
-  #mechanisms. I only implemented this because the
-  #<tt>archive-tar-minitar</tt> gem calls this method when
-  #unpacking a TAR archive from a stream.
-  def rewind
-    # HACK: Wipe all data from io-like’s internal read buffer.
-    # This heavily relies on io-like’s internal structure.
-    # Be always sure to test this when a new version of
-    # io-like is released!
-    __io_like__internal_read_buffer.clear
+    # Note: Querying the underlying IO as early as possible allows to
+    # have Ruby's own IO exceptions to bubble up.
+    if length
+      return nil if eof? # In line with IO#read
+      outbuf.force_encoding(Encoding::BINARY) # As per IO#read docs
 
-    # Forcibly close the XZ stream (internally frees it!)
-    res = XZ::LibLZMA.lzma_end(@lzma_stream.pointer)
-    XZ::LZMAError.raise_if_necessary(res)
+      # The user's request is in decompressed bytes, so it doesn't matter
+      # how much is actually read from the compressed file.
+      if @delegate_io.eof?
+        data   = ""
+        action = XZ::LibLZMA::LZMA_FINISH
+      else
+        data   = @delegate_io.read(XZ::CHUNK_SIZE)
+        action = @delegate_io.eof? ? XZ::LibLZMA::LZMA_FINISH : XZ::LibLZMA::LZMA_RUN
+      end
 
-    # Rewind the wrapped IO
-    begin
-      @delegate_io.rewind
-    rescue => e
-      raise(IOError, "Delegate IO failed to rewind! Original message: #{e.message}")
-    end
+      lzma_code(data, action) { |decompressed| @readbuf << decompressed }
 
-    # Reinitialize everything. Note this doesn’t affect @file as it
-    # is already set and stays so (we don’t pass a filename here,
-    # but rather an IO)
-    initialize(@delegate_io, @memory_limit, @flags)
-  end
+      # If the requested amount has been read, return it.
+      # Also return if EOF has been reached. Note that
+      # String#slice! will clear the string to an empty one
+      # if `length' is greater than the string length.
+      # If EOF is not yet reached, try reading and decompresing
+      # more data.
+      if @readbuf.bytesize >= length || @delegate_io.eof?
+        result = @readbuf.slice!(0, length)
+        @pos += result.bytesize
+        return outbuf.replace(result)
+      else
+        return read(length, outbuf)
+      end
+    else
+      # Read the entire file and decompress it into memory, returning it.
+      while chunk = @delegate_io.read(XZ::CHUNK_SIZE)
+        action = @delegate_io.eof? ? XZ::LibLZMA::LZMA_FINISH : XZ::LibLZMA::LZMA_RUN
+        lzma_code(chunk, action) { |decompressed| @readbuf << decompressed }
+      end
 
-  #NO, you CANNOT seek in this object!!
-  #io-like’s default behaviour is to raise Errno::ESPIPE
-  #when calling a non-defined seek, which is not what some
-  #libraries such as RubyGem’s TarReader expect (they expect
-  #a NoMethodError/NameError instead).
-  undef seek
+      @pos += @readbuf.bytesize
 
-  private
+      # Apply encoding conversion.
+      # First, tag the read data with the external encoding.
+      @readbuf.force_encoding(@external_encoding)
 
-  #Called by io-like’s read methods such as #read. Does the heavy work
-  #of feeding liblzma the compressed data and reading the returned
-  #uncompressed data.
-  def unbuffered_read(length)
-    raise(EOFError, "Input data completely processed!") if @__lzma_finished
+      # Now, transcode it to the internal encoding if that was requested.
+      # Otherwise return it with the external encoding as-is.
+      if @internal_encoding
+        @readbuf.encode!(@internal_encoding, **@transcode_options)
+        outbuf.force_encoding(@internal_encoding)
+      else
+        outbuf.force_encoding(@external_encoding)
+      end
 
-    output_buffer_p = FFI::MemoryPointer.new(length) # User guarantees that this fits into RAM
+      outbuf.replace(@readbuf)
+      @readbuf.clear
+      @readbuf.force_encoding(Encoding::BINARY) # Back to binary mode for further reading
 
-    @lzma_stream[:next_out]  = output_buffer_p
-    @lzma_stream[:avail_out] = output_buffer_p.size
+      return outbuf
+    end
+  end
 
-    loop do
-      # DON’T overwrite any not yet consumed input from any previous
-      # run! Instead, wait until the last input data is entirely
-      # consumed, then provide new data.
-      # TODO: Theoretically, one could move the remaining data to the
-      # beginning of the pointer and fill the rest with new data,
-      # being a tiny bit more performant.
-      if @lzma_stream[:avail_in].zero?
-        compressed_data = @delegate_io.read(@input_buffer_p.size) || "" # nil at EOS → ""
-        @input_buffer_p.write_string(compressed_data)
-        @lzma_stream[:next_in] = @input_buffer_p
-        @lzma_stream[:avail_in] = binary_size(compressed_data)
+  # Abort the current decompression process and reset everything
+  # to the start so that reading from this reader will start over
+  # from the beginning of the compressed data.
+  #
+  # The delegate IO has to support the #rewind method. Otherwise
+  # like IO#rewind.
+  def rewind
+    super
 
-        # Now check if we’re at the last bytes of data and set accordingly the
-        # LZMA-action to carry out (for any subsequent runs until
-        # all input data has been consumed and the above condition
-        # is triggered again).
-        #
-        # The @__lzma_action variable is only used in this method
-        # and is _not_ supposed to be accessed from any other method.
-        if compressed_data.empty?
-          @__lzma_action = XZ::LibLZMA::LZMA_ACTION[:lzma_finish]
-        else
-          @__lzma_action = XZ::LibLZMA::LZMA_ACTION[:lzma_run]
-        end
-      end
+    @readbuf.clear
+    res = XZ::LibLZMA.lzma_stream_decoder(@lzma_stream.to_ptr,
+                                      @memory_limit,
+                                      @allflags)
+    XZ::LZMAError.raise_if_necessary(res)
 
-      res = XZ::LibLZMA.lzma_code(@lzma_stream.pointer, @__lzma_action)
+    0 # Mimic IO#rewind's return value
+  end
 
-      # liblzma signals LZMA_BUF_ERROR when the output buffer is
-      # completely filled, which means we can return now.
-      # When it signals LZMA_STREAM_END, the buffer won’t be filled
-      # completely anymore as the whole input data has been consumed.
-      if res == XZ::LibLZMA::LZMA_RET[:lzma_buf_error]
-        # @lzma_stream[:avail_out] holds the number of free bytes _behind_
-        # the produced output!
-        return output_buffer_p.read_string(output_buffer_p.size - @lzma_stream[:avail_out])
-      elsif res == XZ::LibLZMA::LZMA_RET[:lzma_stream_end]
-        # @__lzma_finished is not supposed to be used outside this method!
-        @__lzma_finished = true
-        return output_buffer_p.read_string(output_buffer_p.size - @lzma_stream[:avail_out])
-      else
-        XZ::LZMAError.raise_if_necessary(res)
-      end
-    end #loop
+  # Like IO#ungetbyte.
+  def ungetbyte(obj)
+    if obj.respond_to? :chr
+      @readbuf.prepend(obj.chr)
+    else
+      @readbuf.prepend(obj.to_s)
+    end
+  end
+
+  # Like IO#ungetc.
+  def ungetc(str)
+    @readbuf.prepend(str)
+  end
+
+  # Returns true if:
+  #
+  # 1. The underlying IO has reached EOF, and
+  # 2. liblzma has returned everything it could make out of that.
+  def eof?
+    @delegate_io.eof? && @readbuf.empty?
+  end
 
-  rescue XZ::LZMAError => e
-    raise(SystemCallError, e.message)
+  # Human-readable description
+  def inspect
+    "<#{self.class} pos=#{@pos} bufsize=#{@readbuf.bytesize} finished=#{@finished} closed=#{closed?} io=#{@delegate_io.inspect}>"
   end
 
 end