lz4-ruby 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.rdoc CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Ruby bindings for {LZ4}[http://code.google.com/p/lz4/].
4
4
 
5
- == Install
5
+ == Installation
6
6
 
7
7
  gem install lz4-ruby
8
8
 
@@ -11,17 +11,30 @@ Ruby bindings for {LZ4}[http://code.google.com/p/lz4/].
11
11
  require 'rubygems'
12
12
  require 'lz4-ruby'
13
13
 
14
+ # compress (fast)
14
15
  compressed = LZ4::compress("hello, world")
15
- # compressed = LZ4::compressHC("hello, world")
16
+
17
+ # compress (high compression)
18
+ compressed = LZ4::compressHC("hello, world")
19
+
20
+ # uncompress
16
21
  uncompressed = LZ4::uncompress(compressed)
17
22
 
23
+ == Benchmark
24
+
25
+ Tested on VirtualBox VM : 2-core, 4GB RAM (Host : Core i5-2520M / 8GB RAM).
26
+
27
+ === {enwik8}[http://mattmahoney.net/dc/enwik8.zip]
28
+ method ratio(bpc) comp.time(ms) uncomp.time(ms)
29
+ -------------------------------------------------------
30
+ lz4 4.559 519.25 182.67
31
+ snappy 4.668 1050.73 257.94
32
+ lzo 4.279 1000.13 574.77
33
+
18
34
  == TODO
19
35
 
20
- * Support compressCtx/compress64kCtx methods.
21
36
  * Write API documents.
22
37
  * Write test codes.
23
- * Benchmark other compression libraries (lzo, snappy...).
24
- * Optimize memory allocation.
25
38
  * Support mswin32 platform.
26
39
 
27
40
  == Copyright
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.0
1
+ 0.1.1
@@ -0,0 +1,137 @@
1
+ require 'benchmark'
2
+ require 'rubygems'
3
+
4
+ CHUNK_SIZE = 8 << 20
5
+ NUM_LOOP = 10
6
+
7
+ class DevNullIO
8
+ def write(arg)
9
+ # Do nothing
10
+ end
11
+ end
12
+
13
+ class StringIO
14
+ def initialize(text)
15
+ @text = text
16
+ @len = text.length
17
+ @pos = 0
18
+ end
19
+
20
+ def read(length)
21
+ return nil if @pos == @len
22
+
23
+ length = @len - @pos if @pos + length > @len
24
+
25
+ result = @text[@pos, length]
26
+ @pos += length
27
+
28
+ return result
29
+ end
30
+ end
31
+
32
+ class CompressorBenchmark
33
+ USAGE = <<EOS
34
+ Usage:
35
+ ruby #{$0} compressor testdata
36
+
37
+ Compressor:
38
+ lz4
39
+ snappy
40
+ lzo
41
+ EOS
42
+
43
+ def initialize(args)
44
+ if args.length != 2
45
+ puts USAGE
46
+ exit 1
47
+ end
48
+
49
+ @signature = args[0]
50
+ @compressor_rb = "compressor_#{@signature}.rb"
51
+
52
+ if !File.file?(@compressor_rb)
53
+ puts "Error: Compressor '#{@compressor_rb}' is not found."
54
+ puts USAGE
55
+ exit 1
56
+ end
57
+
58
+ @testdata_filename = args[1]
59
+ if !File.file?(@testdata_filename)
60
+ puts "Error: testdata `#{@testdata_filename}` is not found."
61
+ puts USAGE
62
+ exit 1
63
+ end
64
+
65
+ @compressed_filename = "#{@testdata_filename}.#{@signature}.compressed"
66
+
67
+ require "./#{@compressor_rb}"
68
+ @compressor = create_compressor(CHUNK_SIZE)
69
+ end
70
+
71
+ def setup_compressed
72
+ `ruby #{@compressor_rb} c #{CHUNK_SIZE} <#{@testdata_filename} >#{@compressed_filename}`
73
+ end
74
+
75
+ def benchmark_compression
76
+ data = nil
77
+ File.open(@testdata_filename) do |file|
78
+ data = file.read(File.size(@testdata_filename))
79
+ end
80
+
81
+ devnull = DevNullIO.new
82
+
83
+ # warm up
84
+ @compressor.compress(StringIO.new(data), devnull)
85
+
86
+ result = Benchmark.measure {
87
+ NUM_LOOP.times { |t| @compressor.compress(StringIO.new(data), devnull) }
88
+ }
89
+
90
+ return result
91
+ end
92
+
93
+ def benchmark_uncompression
94
+ data = nil
95
+ File.open(@compressed_filename) do |file|
96
+ data = file.read(File.size(@compressed_filename))
97
+ end
98
+
99
+ devnull = DevNullIO.new
100
+
101
+ # warm up
102
+ @compressor.uncompress(StringIO.new(data), devnull)
103
+
104
+ result = Benchmark.measure {
105
+ NUM_LOOP.times { |t| @compressor.uncompress(StringIO.new(data), devnull) }
106
+ }
107
+
108
+ return result
109
+ end
110
+
111
+ def show_result(result_comp, result_uncomp)
112
+ orig_size = File.size(@testdata_filename)
113
+ comp_size = File.size(@compressed_filename)
114
+
115
+ ratio = comp_size * 8.0 / orig_size
116
+ comp_time = result_comp.real * 1000.0 / NUM_LOOP
117
+ uncomp_time = result_uncomp.real * 1000.0 / NUM_LOOP
118
+
119
+ puts "method\tratio(bpc)\tcomp.time(ms)\tuncomp.time(ms)"
120
+ puts "-------------------------------------------------------"
121
+ puts "%s\t%.3f\t\t%.2f\t\t%.2f" % [@signature, ratio, comp_time, uncomp_time]
122
+ end
123
+
124
+ def do_benchmark
125
+ setup_compressed()
126
+
127
+ result_comp = benchmark_compression()
128
+ result_uncomp = benchmark_uncompression()
129
+
130
+ show_result(result_comp, result_uncomp)
131
+ end
132
+ end
133
+
134
+ if $0 == __FILE__
135
+ CompressorBenchmark.new(ARGV).do_benchmark
136
+ end
137
+
@@ -0,0 +1,5 @@
1
+ #!/bin/sh
2
+
3
+ ruby bench.rb lz4 $1
4
+ ruby bench.rb snappy $1 | tail -n 1
5
+ ruby bench.rb lzo $1 | tail -n 1
@@ -0,0 +1,71 @@
1
+ class Compressor
2
+ DEFAULT_CHUNK_SIZE = 8 << 20
3
+
4
+ def initialize(chunk_size)
5
+ if chunk_size == nil
6
+ @chunk_size = DEFAULT_CHUNK_SIZE
7
+ else
8
+ @chunk_size = chunk_size
9
+ end
10
+
11
+ require_libs()
12
+ end
13
+
14
+ def compress(infile, outfile)
15
+ loop do
16
+ text = infile.read(@chunk_size)
17
+ break if text == nil || text.length == 0
18
+
19
+ compressed = compress_text(text)
20
+ comp_size = compressed.length
21
+
22
+ outfile.write([comp_size].pack("L"))
23
+ outfile.write(compressed)
24
+ end
25
+ end
26
+
27
+ def uncompress(infile, outfile)
28
+ loop do
29
+ comp_size = infile.read(4)
30
+ break if comp_size == nil || comp_size.length == 0
31
+
32
+ comp_size = comp_size.unpack("L")[0]
33
+ compressed = infile.read(comp_size)
34
+
35
+ text = uncompress_text(compressed)
36
+
37
+ outfile.write(text)
38
+ end
39
+ end
40
+
41
+ def self.unit_driver()
42
+ if !(ARGV.length == 1) && !(ARGV.length == 2 && ARGV[0] == 'c')
43
+ puts <<EOS
44
+ Compress:
45
+ ./#{$0} c <infile >outfile
46
+
47
+ Uncompress:
48
+ ./#{$0} u <infile >outfile
49
+ EOS
50
+ exit 1
51
+ end
52
+
53
+ require 'rubygems'
54
+
55
+ case ARGV[0]
56
+ when 'c'
57
+ chunk_size = nil
58
+ chunk_size = ARGV[1].to_i if ARGV.length == 2
59
+ compressor = create_compressor(chunk_size)
60
+ compressor.compress($stdin, $stdout)
61
+
62
+ when 'u'
63
+ compressor = create_compressor(nil)
64
+ compressor.uncompress($stdin, $stdout)
65
+
66
+ else
67
+ puts "Error: illegal argument '#{ARGV[0]}'"
68
+ exit 1
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,25 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require './compressor.rb'
4
+
5
+ class LZ4Compressor < Compressor
6
+ def require_libs
7
+ require 'lz4-ruby'
8
+ end
9
+
10
+ def compress_text(text)
11
+ return LZ4::compress(text)
12
+ end
13
+
14
+ def uncompress_text(compressed)
15
+ return LZ4::uncompress(compressed)
16
+ end
17
+ end
18
+
19
+ def create_compressor(chunk_size)
20
+ return LZ4Compressor.new(chunk_size)
21
+ end
22
+
23
+ if $0 == __FILE__
24
+ Compressor.unit_driver() { |chunk_size| LZ4Compressor.new(chunk_size) }
25
+ end
@@ -0,0 +1,25 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require './compressor.rb'
4
+
5
+ class LZOCompressor < Compressor
6
+ def require_libs
7
+ require 'lzoruby'
8
+ end
9
+
10
+ def compress_text(text)
11
+ return LZO.compress(text)
12
+ end
13
+
14
+ def uncompress_text(compressed)
15
+ return LZO.decompress(compressed)
16
+ end
17
+ end
18
+
19
+ def create_compressor(chunk_size)
20
+ return LZOCompressor.new(chunk_size)
21
+ end
22
+
23
+ if $0 == __FILE__
24
+ Compressor.unit_driver() { |chunk_size| LZOCompressor.new(chunk_size) }
25
+ end
@@ -0,0 +1,25 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require './compressor.rb'
4
+
5
+ class SnappyCompressor < Compressor
6
+ def require_libs
7
+ require 'snappy'
8
+ end
9
+
10
+ def compress_text(text)
11
+ return Snappy.deflate(text)
12
+ end
13
+
14
+ def uncompress_text(compressed)
15
+ return Snappy.inflate(compressed)
16
+ end
17
+ end
18
+
19
+ def create_compressor(chunk_size)
20
+ return SnappyCompressor.new(chunk_size)
21
+ end
22
+
23
+ if $0 == __FILE__
24
+ Compressor.unit_driver() { |chunk_size| SnappyCompressor.new(chunk_size) }
25
+ end
data/ext/lz4ruby.c CHANGED
@@ -5,6 +5,7 @@
5
5
  typedef int (*CompressFunc)(const char *source, char *dest, int isize);
6
6
 
7
7
  static VALUE lz4;
8
+ static VALUE lz4_error;
8
9
 
9
10
  static VALUE compress(CompressFunc compressor, VALUE self, VALUE source) {
10
11
  const char *src_p = NULL;
@@ -19,16 +20,16 @@ static VALUE compress(CompressFunc compressor, VALUE self, VALUE source) {
19
20
  src_size = RSTRING_LEN(source);
20
21
  buf_size = LZ4_compressBound(src_size);
21
22
 
22
- buf = xmalloc(buf_size + 4);
23
+ result = rb_str_new(NULL, buf_size + 4);
24
+ buf = RSTRING_PTR(result);
25
+
23
26
  buf[0] = (char)((src_size >> 24) & 0xff);
24
27
  buf[1] = (char)((src_size >> 16) & 0xff);
25
28
  buf[2] = (char)((src_size >> 8) & 0xff);
26
29
  buf[3] = (char)(src_size & 0xff);
27
30
 
28
31
  comp_size = compressor(src_p, buf + 4, src_size);
29
- result = rb_str_new(buf, comp_size + 4);
30
-
31
- xfree(buf);
32
+ rb_str_resize(result, comp_size + 4);
32
33
 
33
34
  return result;
34
35
  }
@@ -58,13 +59,15 @@ static VALUE lz4_ruby_uncompress(VALUE self, VALUE source) {
58
59
  | ((src_p[2] & 0xffU) << 8)
59
60
  | (src_p[3] & 0xffU);
60
61
 
61
- buf = xmalloc(buf_size);
62
+ result = rb_str_new(NULL, buf_size + 1);
63
+ buf = RSTRING_PTR(result);
62
64
 
63
65
  read_bytes = LZ4_uncompress(src_p + 4, buf, buf_size);
64
- // TODO read_bytes が負だったら、データが壊れていることを表す
65
- result = rb_str_new(buf, buf_size);
66
+ if (read_bytes < 0) {
67
+ rb_raise(lz4_error, "Compressed data is maybe corrupted.");
68
+ }
66
69
 
67
- xfree(buf);
70
+ buf[buf_size] = '\0';
68
71
 
69
72
  return result;
70
73
  }
@@ -75,4 +78,6 @@ void Init_lz4ruby(void) {
75
78
  rb_define_module_function(lz4, "compress", lz4_ruby_compress, 1);
76
79
  rb_define_module_function(lz4, "compressHC", lz4_ruby_compressHC, 1);
77
80
  rb_define_module_function(lz4, "uncompress", lz4_ruby_uncompress, 1);
81
+
82
+ lz4_error = rb_define_class_under(lz4, "Error", rb_eStandardError);
78
83
  }
data/lz4-ruby.gemspec CHANGED
@@ -5,7 +5,7 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = "lz4-ruby"
8
- s.version = "0.1.0"
8
+ s.version = "0.1.1"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["KOMIYA Atsushi"]
@@ -23,6 +23,12 @@ Gem::Specification.new do |s|
23
23
  "README.rdoc",
24
24
  "Rakefile",
25
25
  "VERSION",
26
+ "benchmarking/bench.rb",
27
+ "benchmarking/bench_all.sh",
28
+ "benchmarking/compressor.rb",
29
+ "benchmarking/compressor_lz4.rb",
30
+ "benchmarking/compressor_lzo.rb",
31
+ "benchmarking/compressor_snappy.rb",
26
32
  "ext/extconf.rb",
27
33
  "ext/lz4ruby.c",
28
34
  "lib/lz4-ruby.rb",
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: lz4-ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -89,6 +89,12 @@ files:
89
89
  - README.rdoc
90
90
  - Rakefile
91
91
  - VERSION
92
+ - benchmarking/bench.rb
93
+ - benchmarking/bench_all.sh
94
+ - benchmarking/compressor.rb
95
+ - benchmarking/compressor_lz4.rb
96
+ - benchmarking/compressor_lzo.rb
97
+ - benchmarking/compressor_snappy.rb
92
98
  - ext/extconf.rb
93
99
  - ext/lz4ruby.c
94
100
  - lib/lz4-ruby.rb
@@ -110,7 +116,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
110
116
  version: '0'
111
117
  segments:
112
118
  - 0
113
- hash: 412764659
119
+ hash: -327617105
114
120
  required_rubygems_version: !ruby/object:Gem::Requirement
115
121
  none: false
116
122
  requirements: