lz4-ruby 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +18 -5
- data/VERSION +1 -1
- data/benchmarking/bench.rb +137 -0
- data/benchmarking/bench_all.sh +5 -0
- data/benchmarking/compressor.rb +71 -0
- data/benchmarking/compressor_lz4.rb +25 -0
- data/benchmarking/compressor_lzo.rb +25 -0
- data/benchmarking/compressor_snappy.rb +25 -0
- data/ext/lz4ruby.c +13 -8
- data/lz4-ruby.gemspec +7 -1
- metadata +8 -2
data/README.rdoc
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Ruby bindings for {LZ4}[http://code.google.com/p/lz4/].
|
4
4
|
|
5
|
-
==
|
5
|
+
== Installation
|
6
6
|
|
7
7
|
gem install lz4-ruby
|
8
8
|
|
@@ -11,17 +11,30 @@ Ruby bindings for {LZ4}[http://code.google.com/p/lz4/].
|
|
11
11
|
require 'rubygems'
|
12
12
|
require 'lz4-ruby'
|
13
13
|
|
14
|
+
# compress (fast)
|
14
15
|
compressed = LZ4::compress("hello, world")
|
15
|
-
|
16
|
+
|
17
|
+
# compress (high compression)
|
18
|
+
compressed = LZ4::compressHC("hello, world")
|
19
|
+
|
20
|
+
# uncompress
|
16
21
|
uncompressed = LZ4::uncompress(compressed)
|
17
22
|
|
23
|
+
== Benchmark
|
24
|
+
|
25
|
+
Tested on VirtualBox VM : 2-core, 4GB RAM (Host : Core i5-2520M / 8GB RAM).
|
26
|
+
|
27
|
+
=== {enwik8}[http://mattmahoney.net/dc/enwik8.zip]
|
28
|
+
method ratio(bpc) comp.time(ms) uncomp.time(ms)
|
29
|
+
-------------------------------------------------------
|
30
|
+
lz4 4.559 519.25 182.67
|
31
|
+
snappy 4.668 1050.73 257.94
|
32
|
+
lzo 4.279 1000.13 574.77
|
33
|
+
|
18
34
|
== TODO
|
19
35
|
|
20
|
-
* Support compressCtx/compress64kCtx methods.
|
21
36
|
* Write API documents.
|
22
37
|
* Write test codes.
|
23
|
-
* Benchmark other compression libraries (lzo, snappy...).
|
24
|
-
* Optimize memory allocation.
|
25
38
|
* Support mswin32 platform.
|
26
39
|
|
27
40
|
== Copyright
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.1
|
@@ -0,0 +1,137 @@
|
|
1
|
+
require 'benchmark'
|
2
|
+
require 'rubygems'
|
3
|
+
|
4
|
+
CHUNK_SIZE = 8 << 20
|
5
|
+
NUM_LOOP = 10
|
6
|
+
|
7
|
+
class DevNullIO
|
8
|
+
def write(arg)
|
9
|
+
# Do nothing
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class StringIO
|
14
|
+
def initialize(text)
|
15
|
+
@text = text
|
16
|
+
@len = text.length
|
17
|
+
@pos = 0
|
18
|
+
end
|
19
|
+
|
20
|
+
def read(length)
|
21
|
+
return nil if @pos == @len
|
22
|
+
|
23
|
+
length = @len - @pos if @pos + length > @len
|
24
|
+
|
25
|
+
result = @text[@pos, length]
|
26
|
+
@pos += length
|
27
|
+
|
28
|
+
return result
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class CompressorBenchmark
|
33
|
+
USAGE = <<EOS
|
34
|
+
Usage:
|
35
|
+
ruby #{$0} compressor testdata
|
36
|
+
|
37
|
+
Compressor:
|
38
|
+
lz4
|
39
|
+
snappy
|
40
|
+
lzo
|
41
|
+
EOS
|
42
|
+
|
43
|
+
def initialize(args)
|
44
|
+
if args.length != 2
|
45
|
+
puts USAGE
|
46
|
+
exit 1
|
47
|
+
end
|
48
|
+
|
49
|
+
@signature = args[0]
|
50
|
+
@compressor_rb = "compressor_#{@signature}.rb"
|
51
|
+
|
52
|
+
if !File.file?(@compressor_rb)
|
53
|
+
puts "Error: Compressor '#{@compressor_rb}' is not found."
|
54
|
+
puts USAGE
|
55
|
+
exit 1
|
56
|
+
end
|
57
|
+
|
58
|
+
@testdata_filename = args[1]
|
59
|
+
if !File.file?(@testdata_filename)
|
60
|
+
puts "Error: testdata `#{@testdata_filename}` is not found."
|
61
|
+
puts USAGE
|
62
|
+
exit 1
|
63
|
+
end
|
64
|
+
|
65
|
+
@compressed_filename = "#{@testdata_filename}.#{@signature}.compressed"
|
66
|
+
|
67
|
+
require "./#{@compressor_rb}"
|
68
|
+
@compressor = create_compressor(CHUNK_SIZE)
|
69
|
+
end
|
70
|
+
|
71
|
+
def setup_compressed
|
72
|
+
`ruby #{@compressor_rb} c #{CHUNK_SIZE} <#{@testdata_filename} >#{@compressed_filename}`
|
73
|
+
end
|
74
|
+
|
75
|
+
def benchmark_compression
|
76
|
+
data = nil
|
77
|
+
File.open(@testdata_filename) do |file|
|
78
|
+
data = file.read(File.size(@testdata_filename))
|
79
|
+
end
|
80
|
+
|
81
|
+
devnull = DevNullIO.new
|
82
|
+
|
83
|
+
# warm up
|
84
|
+
@compressor.compress(StringIO.new(data), devnull)
|
85
|
+
|
86
|
+
result = Benchmark.measure {
|
87
|
+
NUM_LOOP.times { |t| @compressor.compress(StringIO.new(data), devnull) }
|
88
|
+
}
|
89
|
+
|
90
|
+
return result
|
91
|
+
end
|
92
|
+
|
93
|
+
def benchmark_uncompression
|
94
|
+
data = nil
|
95
|
+
File.open(@compressed_filename) do |file|
|
96
|
+
data = file.read(File.size(@compressed_filename))
|
97
|
+
end
|
98
|
+
|
99
|
+
devnull = DevNullIO.new
|
100
|
+
|
101
|
+
# warm up
|
102
|
+
@compressor.uncompress(StringIO.new(data), devnull)
|
103
|
+
|
104
|
+
result = Benchmark.measure {
|
105
|
+
NUM_LOOP.times { |t| @compressor.uncompress(StringIO.new(data), devnull) }
|
106
|
+
}
|
107
|
+
|
108
|
+
return result
|
109
|
+
end
|
110
|
+
|
111
|
+
def show_result(result_comp, result_uncomp)
|
112
|
+
orig_size = File.size(@testdata_filename)
|
113
|
+
comp_size = File.size(@compressed_filename)
|
114
|
+
|
115
|
+
ratio = comp_size * 8.0 / orig_size
|
116
|
+
comp_time = result_comp.real * 1000.0 / NUM_LOOP
|
117
|
+
uncomp_time = result_uncomp.real * 1000.0 / NUM_LOOP
|
118
|
+
|
119
|
+
puts "method\tratio(bpc)\tcomp.time(ms)\tuncomp.time(ms)"
|
120
|
+
puts "-------------------------------------------------------"
|
121
|
+
puts "%s\t%.3f\t\t%.2f\t\t%.2f" % [@signature, ratio, comp_time, uncomp_time]
|
122
|
+
end
|
123
|
+
|
124
|
+
def do_benchmark
|
125
|
+
setup_compressed()
|
126
|
+
|
127
|
+
result_comp = benchmark_compression()
|
128
|
+
result_uncomp = benchmark_uncompression()
|
129
|
+
|
130
|
+
show_result(result_comp, result_uncomp)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
if $0 == __FILE__
|
135
|
+
CompressorBenchmark.new(ARGV).do_benchmark
|
136
|
+
end
|
137
|
+
|
@@ -0,0 +1,71 @@
|
|
1
|
+
class Compressor
|
2
|
+
DEFAULT_CHUNK_SIZE = 8 << 20
|
3
|
+
|
4
|
+
def initialize(chunk_size)
|
5
|
+
if chunk_size == nil
|
6
|
+
@chunk_size = DEFAULT_CHUNK_SIZE
|
7
|
+
else
|
8
|
+
@chunk_size = chunk_size
|
9
|
+
end
|
10
|
+
|
11
|
+
require_libs()
|
12
|
+
end
|
13
|
+
|
14
|
+
def compress(infile, outfile)
|
15
|
+
loop do
|
16
|
+
text = infile.read(@chunk_size)
|
17
|
+
break if text == nil || text.length == 0
|
18
|
+
|
19
|
+
compressed = compress_text(text)
|
20
|
+
comp_size = compressed.length
|
21
|
+
|
22
|
+
outfile.write([comp_size].pack("L"))
|
23
|
+
outfile.write(compressed)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def uncompress(infile, outfile)
|
28
|
+
loop do
|
29
|
+
comp_size = infile.read(4)
|
30
|
+
break if comp_size == nil || comp_size.length == 0
|
31
|
+
|
32
|
+
comp_size = comp_size.unpack("L")[0]
|
33
|
+
compressed = infile.read(comp_size)
|
34
|
+
|
35
|
+
text = uncompress_text(compressed)
|
36
|
+
|
37
|
+
outfile.write(text)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.unit_driver()
|
42
|
+
if !(ARGV.length == 1) && !(ARGV.length == 2 && ARGV[0] == 'c')
|
43
|
+
puts <<EOS
|
44
|
+
Compress:
|
45
|
+
./#{$0} c <infile >outfile
|
46
|
+
|
47
|
+
Uncompress:
|
48
|
+
./#{$0} u <infile >outfile
|
49
|
+
EOS
|
50
|
+
exit 1
|
51
|
+
end
|
52
|
+
|
53
|
+
require 'rubygems'
|
54
|
+
|
55
|
+
case ARGV[0]
|
56
|
+
when 'c'
|
57
|
+
chunk_size = nil
|
58
|
+
chunk_size = ARGV[1].to_i if ARGV.length == 2
|
59
|
+
compressor = create_compressor(chunk_size)
|
60
|
+
compressor.compress($stdin, $stdout)
|
61
|
+
|
62
|
+
when 'u'
|
63
|
+
compressor = create_compressor(nil)
|
64
|
+
compressor.uncompress($stdin, $stdout)
|
65
|
+
|
66
|
+
else
|
67
|
+
puts "Error: illegal argument '#{ARGV[0]}'"
|
68
|
+
exit 1
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require './compressor.rb'
|
4
|
+
|
5
|
+
class LZ4Compressor < Compressor
|
6
|
+
def require_libs
|
7
|
+
require 'lz4-ruby'
|
8
|
+
end
|
9
|
+
|
10
|
+
def compress_text(text)
|
11
|
+
return LZ4::compress(text)
|
12
|
+
end
|
13
|
+
|
14
|
+
def uncompress_text(compressed)
|
15
|
+
return LZ4::uncompress(compressed)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_compressor(chunk_size)
|
20
|
+
return LZ4Compressor.new(chunk_size)
|
21
|
+
end
|
22
|
+
|
23
|
+
if $0 == __FILE__
|
24
|
+
Compressor.unit_driver() { |chunk_size| LZ4Compressor.new(chunk_size) }
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require './compressor.rb'
|
4
|
+
|
5
|
+
class LZOCompressor < Compressor
|
6
|
+
def require_libs
|
7
|
+
require 'lzoruby'
|
8
|
+
end
|
9
|
+
|
10
|
+
def compress_text(text)
|
11
|
+
return LZO.compress(text)
|
12
|
+
end
|
13
|
+
|
14
|
+
def uncompress_text(compressed)
|
15
|
+
return LZO.decompress(compressed)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_compressor(chunk_size)
|
20
|
+
return LZOCompressor.new(chunk_size)
|
21
|
+
end
|
22
|
+
|
23
|
+
if $0 == __FILE__
|
24
|
+
Compressor.unit_driver() { |chunk_size| LZOCompressor.new(chunk_size) }
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require './compressor.rb'
|
4
|
+
|
5
|
+
class SnappyCompressor < Compressor
|
6
|
+
def require_libs
|
7
|
+
require 'snappy'
|
8
|
+
end
|
9
|
+
|
10
|
+
def compress_text(text)
|
11
|
+
return Snappy.deflate(text)
|
12
|
+
end
|
13
|
+
|
14
|
+
def uncompress_text(compressed)
|
15
|
+
return Snappy.inflate(compressed)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_compressor(chunk_size)
|
20
|
+
return SnappyCompressor.new(chunk_size)
|
21
|
+
end
|
22
|
+
|
23
|
+
if $0 == __FILE__
|
24
|
+
Compressor.unit_driver() { |chunk_size| SnappyCompressor.new(chunk_size) }
|
25
|
+
end
|
data/ext/lz4ruby.c
CHANGED
@@ -5,6 +5,7 @@
|
|
5
5
|
typedef int (*CompressFunc)(const char *source, char *dest, int isize);
|
6
6
|
|
7
7
|
static VALUE lz4;
|
8
|
+
static VALUE lz4_error;
|
8
9
|
|
9
10
|
static VALUE compress(CompressFunc compressor, VALUE self, VALUE source) {
|
10
11
|
const char *src_p = NULL;
|
@@ -19,16 +20,16 @@ static VALUE compress(CompressFunc compressor, VALUE self, VALUE source) {
|
|
19
20
|
src_size = RSTRING_LEN(source);
|
20
21
|
buf_size = LZ4_compressBound(src_size);
|
21
22
|
|
22
|
-
|
23
|
+
result = rb_str_new(NULL, buf_size + 4);
|
24
|
+
buf = RSTRING_PTR(result);
|
25
|
+
|
23
26
|
buf[0] = (char)((src_size >> 24) & 0xff);
|
24
27
|
buf[1] = (char)((src_size >> 16) & 0xff);
|
25
28
|
buf[2] = (char)((src_size >> 8) & 0xff);
|
26
29
|
buf[3] = (char)(src_size & 0xff);
|
27
30
|
|
28
31
|
comp_size = compressor(src_p, buf + 4, src_size);
|
29
|
-
result
|
30
|
-
|
31
|
-
xfree(buf);
|
32
|
+
rb_str_resize(result, comp_size + 4);
|
32
33
|
|
33
34
|
return result;
|
34
35
|
}
|
@@ -58,13 +59,15 @@ static VALUE lz4_ruby_uncompress(VALUE self, VALUE source) {
|
|
58
59
|
| ((src_p[2] & 0xffU) << 8)
|
59
60
|
| (src_p[3] & 0xffU);
|
60
61
|
|
61
|
-
|
62
|
+
result = rb_str_new(NULL, buf_size + 1);
|
63
|
+
buf = RSTRING_PTR(result);
|
62
64
|
|
63
65
|
read_bytes = LZ4_uncompress(src_p + 4, buf, buf_size);
|
64
|
-
|
65
|
-
|
66
|
+
if (read_bytes < 0) {
|
67
|
+
rb_raise(lz4_error, "Compressed data is maybe corrupted.");
|
68
|
+
}
|
66
69
|
|
67
|
-
|
70
|
+
buf[buf_size] = '\0';
|
68
71
|
|
69
72
|
return result;
|
70
73
|
}
|
@@ -75,4 +78,6 @@ void Init_lz4ruby(void) {
|
|
75
78
|
rb_define_module_function(lz4, "compress", lz4_ruby_compress, 1);
|
76
79
|
rb_define_module_function(lz4, "compressHC", lz4_ruby_compressHC, 1);
|
77
80
|
rb_define_module_function(lz4, "uncompress", lz4_ruby_uncompress, 1);
|
81
|
+
|
82
|
+
lz4_error = rb_define_class_under(lz4, "Error", rb_eStandardError);
|
78
83
|
}
|
data/lz4-ruby.gemspec
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "lz4-ruby"
|
8
|
-
s.version = "0.1.
|
8
|
+
s.version = "0.1.1"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["KOMIYA Atsushi"]
|
@@ -23,6 +23,12 @@ Gem::Specification.new do |s|
|
|
23
23
|
"README.rdoc",
|
24
24
|
"Rakefile",
|
25
25
|
"VERSION",
|
26
|
+
"benchmarking/bench.rb",
|
27
|
+
"benchmarking/bench_all.sh",
|
28
|
+
"benchmarking/compressor.rb",
|
29
|
+
"benchmarking/compressor_lz4.rb",
|
30
|
+
"benchmarking/compressor_lzo.rb",
|
31
|
+
"benchmarking/compressor_snappy.rb",
|
26
32
|
"ext/extconf.rb",
|
27
33
|
"ext/lz4ruby.c",
|
28
34
|
"lib/lz4-ruby.rb",
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: lz4-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -89,6 +89,12 @@ files:
|
|
89
89
|
- README.rdoc
|
90
90
|
- Rakefile
|
91
91
|
- VERSION
|
92
|
+
- benchmarking/bench.rb
|
93
|
+
- benchmarking/bench_all.sh
|
94
|
+
- benchmarking/compressor.rb
|
95
|
+
- benchmarking/compressor_lz4.rb
|
96
|
+
- benchmarking/compressor_lzo.rb
|
97
|
+
- benchmarking/compressor_snappy.rb
|
92
98
|
- ext/extconf.rb
|
93
99
|
- ext/lz4ruby.c
|
94
100
|
- lib/lz4-ruby.rb
|
@@ -110,7 +116,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
110
116
|
version: '0'
|
111
117
|
segments:
|
112
118
|
- 0
|
113
|
-
hash:
|
119
|
+
hash: -327617105
|
114
120
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
115
121
|
none: false
|
116
122
|
requirements:
|