lz4-ruby 0.1.0 → 0.1.1

data/README.rdoc

@@ -2,7 +2,7 @@
 
 Ruby bindings for {LZ4}[http://code.google.com/p/lz4/].
 
-== Install
+== Installation
 
     gem install lz4-ruby
 
@@ -11,17 +11,30 @@ Ruby bindings for {LZ4}[http://code.google.com/p/lz4/].
     require 'rubygems'
     require 'lz4-ruby'
 
+    # compress (fast)
     compressed = LZ4::compress("hello, world")
-    # compressed = LZ4::compressHC("hello, world")
+
+    # compress (high compression)
+    compressed = LZ4::compressHC("hello, world")
+
+    # uncompress
     uncompressed = LZ4::uncompress(compressed)
 
+== Benchmark
+
+Tested on VirtualBox VM : 2-core, 4GB RAM (Host : Core i5-2520M / 8GB RAM).
+
+=== {enwik8}[http://mattmahoney.net/dc/enwik8.zip]
+    method  ratio(bpc)      comp.time(ms)   uncomp.time(ms)
+    -------------------------------------------------------
+    lz4     4.559           519.25          182.67
+    snappy  4.668           1050.73         257.94
+    lzo     4.279           1000.13         574.77
+
 == TODO
 
-* Support compressCtx/compress64kCtx methods.
 * Write API documents.
 * Write test codes.
-* Benchmark other compression libraries (lzo, snappy...).
-* Optimize memory allocation.
 * Support mswin32 platform.
 
 == Copyright

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/benchmarking/bench.rb

@@ -0,0 +1,137 @@
+require 'benchmark'
+require 'rubygems'
+
+CHUNK_SIZE = 8 << 20
+NUM_LOOP = 10
+
+class DevNullIO
+  def write(arg)
+    # Do nothing
+  end
+end
+
+class StringIO
+  def initialize(text)
+    @text = text
+    @len = text.length
+    @pos = 0
+  end
+
+  def read(length)
+    return nil if @pos == @len
+
+    length = @len - @pos if @pos + length > @len
+    
+    result = @text[@pos, length]
+    @pos += length
+    
+    return result
+  end
+end
+
+class CompressorBenchmark
+  USAGE = <<EOS
+Usage:
+    ruby #{$0} compressor testdata
+
+Compressor:
+    lz4
+    snappy
+    lzo
+EOS
+
+  def initialize(args)
+    if args.length != 2
+      puts USAGE
+      exit 1
+    end
+
+    @signature = args[0]
+    @compressor_rb = "compressor_#{@signature}.rb"
+    
+    if !File.file?(@compressor_rb)
+      puts "Error: Compressor '#{@compressor_rb}' is not found."
+      puts USAGE
+      exit 1
+    end
+
+    @testdata_filename = args[1]
+    if !File.file?(@testdata_filename)
+      puts "Error: testdata `#{@testdata_filename}` is not found."
+      puts USAGE
+      exit 1
+    end
+
+    @compressed_filename = "#{@testdata_filename}.#{@signature}.compressed"
+    
+    require "./#{@compressor_rb}"
+    @compressor = create_compressor(CHUNK_SIZE)
+  end
+
+  def setup_compressed
+    `ruby #{@compressor_rb} c #{CHUNK_SIZE} <#{@testdata_filename} >#{@compressed_filename}`
+  end
+
+  def benchmark_compression
+    data = nil
+    File.open(@testdata_filename) do |file|
+      data = file.read(File.size(@testdata_filename))
+    end
+
+    devnull = DevNullIO.new
+
+    # warm up
+    @compressor.compress(StringIO.new(data), devnull)
+
+    result = Benchmark.measure {
+      NUM_LOOP.times { |t| @compressor.compress(StringIO.new(data), devnull) }
+    }
+
+    return result
+  end
+  
+  def benchmark_uncompression
+    data = nil
+    File.open(@compressed_filename) do |file|
+      data = file.read(File.size(@compressed_filename))
+    end
+
+    devnull = DevNullIO.new
+
+    # warm up
+    @compressor.uncompress(StringIO.new(data), devnull)
+
+    result = Benchmark.measure {
+      NUM_LOOP.times { |t| @compressor.uncompress(StringIO.new(data), devnull) }
+    }
+
+    return result
+  end
+
+  def show_result(result_comp, result_uncomp)
+    orig_size = File.size(@testdata_filename)
+    comp_size = File.size(@compressed_filename)
+    
+    ratio = comp_size * 8.0 / orig_size
+    comp_time = result_comp.real * 1000.0 / NUM_LOOP
+    uncomp_time = result_uncomp.real * 1000.0 / NUM_LOOP
+
+    puts "method\tratio(bpc)\tcomp.time(ms)\tuncomp.time(ms)"
+    puts "-------------------------------------------------------"
+    puts "%s\t%.3f\t\t%.2f\t\t%.2f" % [@signature, ratio, comp_time, uncomp_time]
+  end
+
+  def do_benchmark
+    setup_compressed()
+
+    result_comp = benchmark_compression()
+    result_uncomp = benchmark_uncompression()
+
+    show_result(result_comp, result_uncomp)
+  end
+end
+
+if $0 == __FILE__
+  CompressorBenchmark.new(ARGV).do_benchmark
+end
+

data/benchmarking/bench_all.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+ruby bench.rb lz4 $1
+ruby bench.rb snappy $1 | tail -n 1
+ruby bench.rb lzo $1 | tail -n 1

data/benchmarking/compressor.rb

@@ -0,0 +1,71 @@
+class Compressor
+  DEFAULT_CHUNK_SIZE = 8 << 20
+  
+  def initialize(chunk_size)
+    if chunk_size == nil
+      @chunk_size = DEFAULT_CHUNK_SIZE
+    else
+      @chunk_size = chunk_size
+    end
+    
+    require_libs()
+  end
+
+  def compress(infile, outfile)
+    loop do
+      text = infile.read(@chunk_size)
+      break if text == nil || text.length == 0
+
+      compressed = compress_text(text)
+      comp_size = compressed.length
+
+      outfile.write([comp_size].pack("L"))
+      outfile.write(compressed)
+    end
+  end
+
+  def uncompress(infile, outfile)
+    loop do
+      comp_size = infile.read(4)
+      break if comp_size == nil || comp_size.length == 0
+
+      comp_size = comp_size.unpack("L")[0]
+      compressed = infile.read(comp_size)
+
+      text = uncompress_text(compressed)
+
+      outfile.write(text)
+    end
+  end
+
+  def self.unit_driver()
+    if !(ARGV.length == 1) && !(ARGV.length == 2 && ARGV[0] == 'c')
+      puts <<EOS
+Compress:
+    ./#{$0} c <infile >outfile
+
+Uncompress:
+    ./#{$0} u <infile >outfile
+EOS
+      exit 1
+    end
+  
+    require 'rubygems'
+
+    case ARGV[0]
+    when 'c'
+      chunk_size = nil
+      chunk_size = ARGV[1].to_i if ARGV.length == 2
+      compressor = create_compressor(chunk_size)
+      compressor.compress($stdin, $stdout)
+    
+    when 'u'
+      compressor = create_compressor(nil)
+      compressor.uncompress($stdin, $stdout)
+
+    else
+      puts "Error: illegal argument '#{ARGV[0]}'"
+      exit 1
+    end
+  end
+end

data/benchmarking/compressor_lz4.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require './compressor.rb'
+
+class LZ4Compressor < Compressor
+  def require_libs
+    require 'lz4-ruby'
+  end
+
+  def compress_text(text)
+    return LZ4::compress(text)
+  end
+
+  def uncompress_text(compressed)
+    return LZ4::uncompress(compressed)
+  end
+end
+
+def create_compressor(chunk_size)
+  return LZ4Compressor.new(chunk_size)
+end
+
+if $0 == __FILE__
+  Compressor.unit_driver() { |chunk_size| LZ4Compressor.new(chunk_size) }
+end

data/benchmarking/compressor_lzo.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require './compressor.rb'
+
+class LZOCompressor < Compressor
+  def require_libs
+    require 'lzoruby'
+  end
+
+  def compress_text(text)
+    return LZO.compress(text)
+  end
+
+  def uncompress_text(compressed)
+    return LZO.decompress(compressed)
+  end
+end
+
+def create_compressor(chunk_size)
+  return LZOCompressor.new(chunk_size)
+end
+
+if $0 == __FILE__
+  Compressor.unit_driver() { |chunk_size| LZOCompressor.new(chunk_size) }
+end

data/benchmarking/compressor_snappy.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require './compressor.rb'
+
+class SnappyCompressor < Compressor
+  def require_libs
+    require 'snappy'
+  end
+
+  def compress_text(text)
+    return Snappy.deflate(text)
+  end
+
+  def uncompress_text(compressed)
+    return Snappy.inflate(compressed)
+  end
+end
+
+def create_compressor(chunk_size)
+  return SnappyCompressor.new(chunk_size)
+end
+
+if $0 == __FILE__
+  Compressor.unit_driver() { |chunk_size| SnappyCompressor.new(chunk_size) }
+end

data/ext/lz4ruby.c

@@ -5,6 +5,7 @@
 typedef int (*CompressFunc)(const char *source, char *dest, int isize);
 
 static VALUE lz4;
+static VALUE lz4_error;
 
 static VALUE compress(CompressFunc compressor, VALUE self, VALUE source) {
   const char *src_p = NULL;
@@ -19,16 +20,16 @@ static VALUE compress(CompressFunc compressor, VALUE self, VALUE source) {
   src_size = RSTRING_LEN(source);
   buf_size = LZ4_compressBound(src_size);
 
-  buf = xmalloc(buf_size + 4);
+  result = rb_str_new(NULL, buf_size + 4);
+  buf = RSTRING_PTR(result);
+
   buf[0] = (char)((src_size >> 24) & 0xff);
   buf[1] = (char)((src_size >> 16) & 0xff);
   buf[2] = (char)((src_size >> 8) & 0xff);
   buf[3] = (char)(src_size & 0xff);
 
   comp_size = compressor(src_p, buf + 4, src_size);
-  result = rb_str_new(buf, comp_size + 4);
-
-  xfree(buf);
+  rb_str_resize(result, comp_size + 4);
 
   return result;
 }
@@ -58,13 +59,15 @@ static VALUE lz4_ruby_uncompress(VALUE self, VALUE source) {
     | ((src_p[2] & 0xffU) << 8)
     | (src_p[3] & 0xffU);
 
-  buf = xmalloc(buf_size);
+  result = rb_str_new(NULL, buf_size + 1);
+  buf = RSTRING_PTR(result);
 
   read_bytes = LZ4_uncompress(src_p + 4, buf, buf_size);
-  // TODO read_bytes が負だったら、データが壊れていることを表す
-  result = rb_str_new(buf, buf_size);
+  if (read_bytes < 0) {
+    rb_raise(lz4_error, "Compressed data is maybe corrupted.");
+  }
 
-  xfree(buf);
+  buf[buf_size] = '\0';
 
   return result;
 }
@@ -75,4 +78,6 @@ void Init_lz4ruby(void) {
   rb_define_module_function(lz4, "compress", lz4_ruby_compress, 1);
   rb_define_module_function(lz4, "compressHC", lz4_ruby_compressHC, 1);
   rb_define_module_function(lz4, "uncompress", lz4_ruby_uncompress, 1);
+
+  lz4_error = rb_define_class_under(lz4, "Error", rb_eStandardError);
 }

data/lz4-ruby.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = "lz4-ruby"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["KOMIYA Atsushi"]
@@ -23,6 +23,12 @@ Gem::Specification.new do |s|
     "README.rdoc",
     "Rakefile",
     "VERSION",
+    "benchmarking/bench.rb",
+    "benchmarking/bench_all.sh",
+    "benchmarking/compressor.rb",
+    "benchmarking/compressor_lz4.rb",
+    "benchmarking/compressor_lzo.rb",
+    "benchmarking/compressor_snappy.rb",
     "ext/extconf.rb",
     "ext/lz4ruby.c",
     "lib/lz4-ruby.rb",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lz4-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -89,6 +89,12 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
+- benchmarking/bench.rb
+- benchmarking/bench_all.sh
+- benchmarking/compressor.rb
+- benchmarking/compressor_lz4.rb
+- benchmarking/compressor_lzo.rb
+- benchmarking/compressor_snappy.rb
 - ext/extconf.rb
 - ext/lz4ruby.c
 - lib/lz4-ruby.rb
@@ -110,7 +116,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 412764659
+      hash: -327617105
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: