lz4-ruby 0.2.0 → 0.3.0

data/ext/lz4ruby/lz4hc.h

@@ -1,7 +1,7 @@
 /*
    LZ4 HC - High Compression Mode of LZ4
    Header File
-   Copyright (C) 2011-2012, Yann Collet.
+   Copyright (C) 2011-2013, Yann Collet.
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
@@ -39,19 +39,131 @@ extern "C" {
 #endif
 
 
-int LZ4_compressHC (const char* source, char* dest, int isize);
-
+int LZ4_compressHC (const char* source, char* dest, int inputSize);
 /*
 LZ4_compressHC :
-	return : the number of bytes in compressed buffer dest
-	note : destination buffer must be already allocated. 
-		To avoid any problem, size it to handle worst cases situations (input data not compressible)
-		Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
+    return : the number of bytes in compressed buffer dest
+             or 0 if compression fails.
+    note : destination buffer must be already allocated.
+        To avoid any problem, size it to handle worst cases situations (input data not compressible)
+        Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
+*/
+
+int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+/*
+LZ4_compress_limitedOutput() :
+    Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
+    If it cannot achieve it, compression will stop, and result of the function will be zero.
+    This function never writes outside of provided output buffer.
+
+    inputSize  : Max supported value is 1 GB
+    maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
+    return : the number of output bytes written in buffer 'dest'
+             or 0 if compression fails.
 */
 
 
+int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+/*
+    Same functions as above, but with programmable 'compressionLevel'.
+    Recommended values are between 4 and 9, although any value between 0 and 16 will work.
+    'compressionLevel'==0 means use default 'compressionLevel' value.
+    Values above 16 behave the same as 16.
+    Equivalent variants exist for all other compression functions below.
+*/
+
 /* Note :
-Decompression functions are provided within regular LZ4 source code (see "lz4.h") (BSD license)
+Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
+*/
+
+
+/**************************************
+   Using an external allocation
+**************************************/
+int LZ4_sizeofStateHC();
+int LZ4_compressHC_withStateHC               (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+int LZ4_compressHC2_withStateHC              (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+
+/*
+These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
+To know how much memory must be allocated for the compression tables, use :
+int LZ4_sizeofStateHC();
+
+Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0).
+
+The allocated memory can be provided to the compressions functions using 'void* state' parameter.
+LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions.
+They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
+*/
+
+
+/**************************************
+   Streaming Functions
+**************************************/
+void* LZ4_createHC (const char* inputBuffer);
+int   LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
+int   LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
+char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+int   LZ4_freeHC (void* LZ4HC_Data);
+
+int   LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+int   LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+
+/*
+These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
+In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
+
+void* LZ4_createHC (const char* inputBuffer);
+The result of the function is the (void*) pointer on the LZ4HC Data Structure.
+This pointer will be needed in all other functions.
+If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
+The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
+To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
+Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
+but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
+If next block does not begin immediately after the previous one, the compression will fail (return 0).
+
+When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
+char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
+must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
+Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
+==> The memory position where the next input data block must start is provided as the result of the function.
+
+Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.
+
+When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
+*/
+
+int LZ4_sizeofStreamStateHC();
+int LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+
+/*
+These functions achieve the same result as :
+void* LZ4_createHC (const char* inputBuffer);
+
+They are provided here to allow the user program to allocate memory using its own routines.
+
+To know how much space must be allocated, use LZ4_sizeofStreamStateHC();
+Note also that space must be aligned for pointers (32 or 64 bits).
+
+Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+void* state is a pointer to the space allocated.
+It must be aligned for pointers (32 or 64 bits), and be large enough.
+The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
+return value of LZ4_resetStreamStateHC() must be 0 is OK.
+Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)).
 */
 
 

data/ext/lz4ruby/lz4ruby.c

@@ -3,6 +3,7 @@
 #include "lz4hc.h"
 
 typedef int (*CompressFunc)(const char *source, char *dest, int isize);
+typedef int (*CompressLimitedOutputFunc)(const char* source, char* dest, int inputSize, int maxOutputSize);
 
 static VALUE lz4internal;
 static VALUE lz4_error;
@@ -43,6 +44,80 @@ static VALUE compress_internal(CompressFunc compressor, VALUE header, VALUE inpu
   return result;
 }
 
+static VALUE compress_raw_internal(
+    CompressLimitedOutputFunc compressor,
+    VALUE _input,
+    VALUE _input_size,
+    VALUE _output_buffer,
+    VALUE _max_output_size) {
+
+  const char *src_p;
+  int src_size;
+
+  int needs_resize;
+  char *buf_p;
+
+  int max_output_size;
+
+  int comp_size;
+
+
+  Check_Type(_input, T_STRING);
+  src_p = RSTRING_PTR(_input);
+  src_size = NUM2INT(_input_size);
+
+  if (NIL_P(_output_buffer)) {
+    needs_resize = 1;
+    _output_buffer = rb_str_new(NULL, _max_output_size);
+
+  } else {
+    needs_resize = 0;
+  }
+
+  buf_p = RSTRING_PTR(_output_buffer);
+
+  max_output_size = NUM2INT(_max_output_size);
+
+  comp_size = compressor(src_p, buf_p, src_size, max_output_size);
+
+  if (comp_size > 0 && needs_resize) {
+    rb_str_resize(_output_buffer, comp_size);
+  }
+
+  return rb_ary_new3(2, _output_buffer, INT2NUM(comp_size));
+}
+
+static VALUE lz4internal_compress_raw(
+    VALUE self,
+    VALUE _input,
+    VALUE _input_size,
+    VALUE _output_buffer,
+    VALUE _max_output_size)
+  {
+    return compress_raw_internal(
+        LZ4_compress_limitedOutput,
+	_input,
+	_input_size,
+	_output_buffer,
+	_max_output_size);
+}
+
+static VALUE lz4internal_compressHC_raw(
+    VALUE self,
+    VALUE _input,
+    VALUE _input_size,
+    VALUE _output_buffer,
+    VALUE _max_output_size)
+  {
+    return compress_raw_internal(
+        LZ4_compressHC_limitedOutput,
+	_input,
+	_input_size,
+	_output_buffer,
+	_max_output_size);
+}
+					
+
 static VALUE lz4internal_compress(VALUE self, VALUE header, VALUE input, VALUE in_size) {
   return compress_internal(LZ4_compress, header, input, in_size);
 }
@@ -81,6 +156,202 @@ static VALUE lz4internal_uncompress(VALUE self, VALUE input, VALUE in_size, VALU
   return result;
 }
 
+static VALUE lz4internal_decompress_raw(
+    VALUE self,
+    VALUE _input,
+    VALUE _input_size,
+    VALUE _output_buffer,
+    VALUE _max_output_size) {
+
+  const char *src_p;
+  int src_size;
+
+  int max_output_size;
+
+  int needs_resize;
+  char *buf_p;
+
+  int decomp_size;
+
+  Check_Type(_input, T_STRING);
+  src_p = RSTRING_PTR(_input);
+  src_size = NUM2INT(_input_size);
+
+  max_output_size = NUM2INT(_max_output_size);
+
+  if (NIL_P(_output_buffer)) {
+    needs_resize = 1;
+    _output_buffer = rb_str_new(NULL, max_output_size);
+
+  } else {
+    needs_resize = 0;
+  }
+
+  buf_p = RSTRING_PTR(_output_buffer);
+  decomp_size = LZ4_decompress_safe(src_p, buf_p, src_size, max_output_size);
+
+  if (decomp_size > 0 && needs_resize) {
+    rb_str_resize(_output_buffer, decomp_size);
+  }
+
+  return rb_ary_new3(2, _output_buffer, INT2NUM(decomp_size));
+}
+
+#if 0
+
+static inline void lz4internal_raw_compress_scanargs(int argc, VALUE *argv, VALUE *src, VALUE *dest, size_t *srcsize, size_t *maxsize) {
+  switch (argc) {
+  case 1:
+    *src = argv[0];
+    Check_Type(*src, RUBY_T_STRING);
+    *srcsize = RSTRING_LEN(*src);
+    *dest = rb_str_buf_new(0);
+    *maxsize = LZ4_compressBound(*srcsize);
+    break;
+  case 2:
+    *src = argv[0];
+    Check_Type(*src, RUBY_T_STRING);
+    *srcsize = RSTRING_LEN(*src);
+    if (TYPE(argv[1]) == T_STRING) {
+      *dest = argv[1];
+      *maxsize = LZ4_compressBound(*srcsize);
+    } else {
+      *dest = rb_str_buf_new(0);
+      *maxsize = NUM2SIZET(argv[1]);
+    }
+    break;
+  case 3:
+    *src = argv[0];
+    Check_Type(*src, RUBY_T_STRING);
+    *srcsize = RSTRING_LEN(*src);
+    *dest = argv[1];
+    Check_Type(*dest, RUBY_T_STRING);
+    *maxsize = NUM2SIZET(argv[2]);
+    break;
+  default:
+    //rb_error_arity(argc, 1, 3);
+    rb_scan_args(argc, argv, "12", NULL, NULL, NULL);
+    // the following code is used to eliminate compiler warnings.
+    *src = *dest = 0;
+    *srcsize = *maxsize = 0;
+  }
+}
+
+static inline VALUE lz4internal_raw_compress_common(int argc, VALUE *argv, VALUE lz4, CompressLimitedOutputFunc compressor) {
+  VALUE src, dest;
+  size_t srcsize;
+  size_t maxsize;
+
+  lz4internal_raw_compress_scanargs(argc, argv, &src, &dest, &srcsize, &maxsize);
+
+  if (srcsize > LZ4_MAX_INPUT_SIZE) {
+    rb_raise(lz4_error,
+             "input size is too big for lz4 compress (max %u bytes)",
+             LZ4_MAX_INPUT_SIZE);
+  }
+  rb_str_modify(dest);
+  rb_str_resize(dest, maxsize);
+  rb_str_set_len(dest, 0);
+
+  int size = compressor(RSTRING_PTR(src), RSTRING_PTR(dest), srcsize, maxsize);
+  if (size < 0) {
+    rb_raise(lz4_error, "failed LZ4 raw compress");
+  }
+
+  rb_str_resize(dest, size);
+  rb_str_set_len(dest, size);
+
+  return dest;
+}
+
+/*
+ * call-seq:
+ *  (compressed string data)  raw_compress(src)
+ *  (compressed string data)  raw_compress(src, max_dest_size)
+ *  (dest with compressed string data)  raw_compress(src, dest)
+ *  (dest with compressed string data)  raw_compress(src, dest, max_dest_size)
+ */
+static VALUE lz4internal_raw_compress(int argc, VALUE *argv, VALUE lz4i) {
+  return lz4internal_raw_compress_common(argc, argv, lz4i, LZ4_compress_limitedOutput);
+}
+
+/*
+ * call-seq:
+ *  (compressed string data)  raw_compressHC(src)
+ *  (compressed string data)  raw_compressHC(src, max_dest_size)
+ *  (dest with compressed string data)  raw_compressHC(src, dest)
+ *  (dest with compressed string data)  raw_compressHC(src, dest, max_dest_size)
+ */
+static VALUE lz4internal_raw_compressHC(int argc, VALUE *argv, VALUE lz4i) {
+  return lz4internal_raw_compress_common(argc, argv, lz4i, LZ4_compressHC_limitedOutput);
+}
+
+enum {
+  LZ4RUBY_UNCOMPRESS_MAXSIZE = 1 << 24, // tentative value
+};
+
+static inline void lz4internal_raw_uncompress_scanargs(int argc, VALUE *argv, VALUE *src, VALUE *dest, size_t *maxsize) {
+  switch (argc) {
+  case 1:
+    *src = argv[0];
+    Check_Type(*src, RUBY_T_STRING);
+    *dest = rb_str_buf_new(0);
+    *maxsize = LZ4RUBY_UNCOMPRESS_MAXSIZE;
+    break;
+  case 2:
+    *src = argv[0];
+    Check_Type(*src, RUBY_T_STRING);
+    *dest = argv[1];
+    if (TYPE(*dest) == T_STRING) {
+      *maxsize = LZ4RUBY_UNCOMPRESS_MAXSIZE;
+    } else {
+      *maxsize = NUM2SIZET(*dest);
+      *dest = rb_str_buf_new(0);
+    }
+    break;
+  case 3:
+    *src = argv[0];
+    Check_Type(*src, RUBY_T_STRING);
+    *dest = argv[1];
+    Check_Type(*dest, RUBY_T_STRING);
+    *maxsize = NUM2SIZET(argv[2]);
+    break;
+  default:
+    //rb_error_arity(argc, 2, 3);
+    rb_scan_args(argc, argv, "21", NULL, NULL, NULL);
+    // the following code is used to eliminate compiler warnings.
+    *src = *dest = 0;
+    *maxsize = 0;
+  }
+}
+
+/*
+ * call-seq:
+ *  (uncompressed string data)  raw_uncompress(src, max_dest_size = 1 << 24)
+ *  (dest for uncompressed string data)  raw_uncompress(src, dest, max_dest_size = 1 << 24)
+ */
+static VALUE lz4internal_raw_uncompress(int argc, VALUE *argv, VALUE lz4i) {
+  VALUE src, dest;
+  size_t maxsize;
+  lz4internal_raw_uncompress_scanargs(argc, argv, &src, &dest, &maxsize);
+
+  rb_str_modify(dest);
+  rb_str_resize(dest, maxsize);
+  rb_str_set_len(dest, 0);
+
+  int size = LZ4_decompress_safe(RSTRING_PTR(src), RSTRING_PTR(dest), RSTRING_LEN(src), maxsize);
+  if (size < 0) {
+    rb_raise(lz4_error, "failed LZ4 raw uncompress at %d", -size);
+  }
+
+  rb_str_resize(dest, size);
+  rb_str_set_len(dest, size);
+
+  return dest;
+}
+
+#endif
+
 void Init_lz4ruby(void) {
   lz4internal = rb_define_module("LZ4Internal");
 
@@ -88,5 +359,13 @@ void Init_lz4ruby(void) {
   rb_define_module_function(lz4internal, "compressHC", lz4internal_compressHC, 3);
   rb_define_module_function(lz4internal, "uncompress", lz4internal_uncompress, 4);
 
+  //rb_define_module_function(lz4internal, "raw_compress", lz4internal_raw_compress, -1);
+  //rb_define_module_function(lz4internal, "raw_compressHC", lz4internal_raw_compressHC, -1);
+  //rb_define_module_function(lz4internal, "raw_uncompress", lz4internal_raw_uncompress, -1);
+
+  rb_define_module_function(lz4internal, "compress_raw", lz4internal_compress_raw, 4);
+  rb_define_module_function(lz4internal, "compressHC_raw", lz4internal_compressHC_raw, 4);
+  rb_define_module_function(lz4internal, "decompress_raw", lz4internal_decompress_raw, 4);
+
   lz4_error = rb_define_class_under(lz4internal, "Error", rb_eStandardError);
 }

data/lib/lz4-ruby.rb

@@ -17,28 +17,35 @@ class LZ4
     return _compress(input, in_size, true)
   end
   
-  def self._compress(input, in_size, high_compression)
+  def self.decompress(input, in_size = nil)
     in_size = input.length if in_size == nil
-    header = encode_varbyte(in_size)
+    out_size, varbyte_len = decode_varbyte(input)
 
-    if high_compression
-      return LZ4Internal.compressHC(header, input, in_size)
-    else
-      return LZ4Internal.compress(header, input, in_size)
+    if out_size < 0 || varbyte_len < 0
+      raise "Compressed data is maybe corrupted"
     end
+    
+    return LZ4Internal::uncompress(input, in_size, varbyte_len, out_size)
   end
 
+  # @deprecated Use {#decompress} and will be removed.
   def self.uncompress(input, in_size = nil)
+    return decompress(input, in_size)
+  end
+
+  private
+  def self._compress(input, in_size, high_compression)
     in_size = input.length if in_size == nil
-    out_size, varbyte_len = decode_varbyte(input)
+    header = encode_varbyte(in_size)
 
-    if out_size < 0 || varbyte_len < 0
-      raise "Compressed data is maybe corrupted"
+    if high_compression
+      return LZ4Internal.compressHC(header, input, in_size)
+    else
+      return LZ4Internal.compress(header, input, in_size)
     end
-    
-    return LZ4Internal::uncompress(input, in_size, varbyte_len, out_size)
   end
 
+  private
   def self.encode_varbyte(val)
     varbytes = []
 
@@ -57,6 +64,7 @@ class LZ4
     return varbytes.pack("C*")
   end
 
+  private
   def self.decode_varbyte(text)
     len = [text.length, 5].min
     bytes = text[0, len].unpack("C*")
@@ -71,4 +79,111 @@ class LZ4
 
     return -1, -1
   end
+
+  # Handles LZ4 native data stream (without any additional headers).
+  class Raw
+    # Compresses `source` string.
+    #
+    # @param [String] source string to be compressed
+    # @param [Hash] options
+    # @option options [Fixnum] :input_size length of source to compress (must be less than or equal to `source.length`)
+    # @option options [String] :dest output buffer which will receive compressed string
+    # @option options [Fixnum] :max_output_size acceptable maximum output size
+    # @return [String, Fixnum] compressed string and its length.
+    def self.compress(source, options = {})
+      return _compress(source, false, options)
+    end
+
+    # Compresses `source` string using High Compress Mode.
+    #
+    # @param [String] source string to be compressed
+    # @param [Hash] options
+    # @option options [Fixnum] :input_size length of source to compress (must be less than or equal to `source.length`)
+    # @option options [String] :dest output buffer which will receive compressed string
+    # @option options [Fixnum] :max_output_size acceptable maximum output size
+    # @return [String, Fixnum] compressed string and its length.
+    def self.compressHC(source, options = {})
+      return _compress(source, true, options)
+    end
+
+    private 
+    def self._compress(source, high_compression, options = {})
+      input_size = options[:input_size]
+      if input_size == nil
+        input_size = source.length
+
+      else
+        if source.length < input_size
+          raise ArgumentError, "`:input_size` (#{input_size}) must be less than or equal `source.length` (#{source.length})"
+        end
+      end
+
+      dest = options[:dest]
+
+      max_output_size = options[:max_output_size]
+      if max_output_size == nil
+        if dest != nil
+          max_output_size = dest.length
+        else
+          max_output_size = input_size + (input_size / 255) + 16 if dest == nil
+        end
+
+      else
+        if dest != nil && dest.length < max_output_size
+          raise ArgumentError, "`:dest` buffer size (#{dest.length}) must be greater than or equal `:max_output_size` (#{max_output_size})"
+        end
+      end
+
+      result = nil
+      if high_compression
+        result = LZ4Internal.compressHC_raw(source, input_size, dest, max_output_size)
+      else
+        result = LZ4Internal.compress_raw(source, input_size, dest, max_output_size)
+      end
+
+      if result[1] <= 0
+        raise LZ4Error, "compression failed"
+      end
+
+      return result[0], result[1]
+    end
+
+    # Decompresses `source` compressed string.
+    #
+    # @param [String] source
+    # @param [Fixnum] max_output_size
+    # @param [Hash] options
+    # @option options [Fixnum] :input_size length of source to decompress (must be less than or equal to `source.length`)
+    # @option options [String] :dest output buffer which will receive decompressed string
+    # @return [String, Fixnum] decompressed string and its length.
+    def self.decompress(source, max_output_size, options = {})
+      input_size = options[:input_size]
+      if input_size == nil
+        input_size = source.length
+
+      else
+        if source.length < input_size
+          raise ArgumentError, "`:input_size` (#{input_size}) must be less than or equal `source.length` (#{source.length})"
+        end
+      end
+
+      dest = options[:dest]
+
+      if dest != nil && dest.length < max_output_size
+        raise ArgumentError, "`:dest` buffer size (#{dest.length}) must be greater than or equal `max_output_size` (#{max_output_size})"
+      end
+
+      result = LZ4Internal.decompress_raw(source, input_size, dest, max_output_size)
+
+      if result[1] <= 0
+        return "", 0 if source == "\x00"
+        raise LZ4Error, "decompression failed"
+      end
+
+      return result[0], result[1]
+    end
+  end
+end
+
+class LZ4Error < StandardError
 end