lzo 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.rspec +2 -0
- data/.rubocop.yml +20 -0
- data/.ruby-version +1 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/README.md +46 -0
- data/Rakefile +7 -0
- data/TODO.md +7 -0
- data/lib/lzo.rb +48 -0
- data/lib/lzo/ffi.rb +47 -0
- data/lib/lzo/lzop.rb +13 -0
- data/lib/lzo/lzop/block.rb +47 -0
- data/lib/lzo/lzop/header.rb +32 -0
- data/lib/lzo/lzop/header_flags.rb +48 -0
- data/lib/lzo/lzop/lzo_method.rb +19 -0
- data/lib/lzo/lzop/lzop_compressor.rb +57 -0
- data/lib/lzo/lzop/lzop_decompressor.rb +81 -0
- data/lib/lzo/multi_io.rb +28 -0
- data/lib/lzo/raw.rb +76 -0
- data/lib/lzo/version.rb +4 -0
- data/lzo.gemspec +32 -0
- metadata +219 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5cf3131e88d5a3d3b9032d47c6341ea283901300
|
4
|
+
data.tar.gz: 9ce3f056b7dc3592b899ab8be6b90508c6670298
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8b7e18948dceae6587733c0ce3184964df2c4b58e019fd963428feac19077566c388d07691c7e4a93ae122e41ec80944d0028b75027e16e02e0231b0dcb045a2
|
7
|
+
data.tar.gz: 70c9389aaba4a5ba0cce57195695a50bff91f510ce98712c4ea06732880e8e870e3873a0062718708973ee6c1165f31bb6f157e572bf32b65a84c4bb325a929c
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
AllCops:
|
2
|
+
DisplayCopNames: true
|
3
|
+
|
4
|
+
Metrics/LineLength:
|
5
|
+
Max: 120
|
6
|
+
|
7
|
+
Metrics/MethodLength:
|
8
|
+
Max: 15
|
9
|
+
|
10
|
+
Style/SpaceBeforeBlockBraces:
|
11
|
+
Enabled: false
|
12
|
+
|
13
|
+
Style/SpaceInsideBlockBraces:
|
14
|
+
Enabled: false
|
15
|
+
|
16
|
+
Style/Documentation:
|
17
|
+
Enabled: false
|
18
|
+
|
19
|
+
Style/SingleLineBlockParams:
|
20
|
+
Enabled: false
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.3
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# LZO Ruby gem
|
2
|
+
|
3
|
+
Wikipedia:
|
4
|
+
|
5
|
+
> Lempel–Ziv–Oberhumer (LZO) is a lossless data compression algorithm that is focused on decompression speed.
|
6
|
+
|
7
|
+
It's an alternative to Gzip, essentially. This gem exists because the previous
|
8
|
+
Ruby LZO gem ([`lzoruby`][lzoruby]) hasn't been updated for 5 years and doesn't
|
9
|
+
support LZOP (as generated by the `lzop` utility) or JRuby. We use the
|
10
|
+
[`ffi`][ffi] gem to provide VM-agnostic bindings to the LZO library.
|
11
|
+
|
12
|
+
[lzoruby]: https://rubygems.org/gems/lzoruby
|
13
|
+
[ffi]: https://rubygems.org/gems/ffi
|
14
|
+
|
15
|
+
## Installation
|
16
|
+
|
17
|
+
The LZO library is **not** bundled with this gem. It must be installed beforehand.
|
18
|
+
Here's how you can do that on various popular OSes:
|
19
|
+
|
20
|
+
* OS X: `brew install lzo`
|
21
|
+
* Ubuntu: `apt-get install liblzo2-2` (`liblzo2-dev` works fine too)
|
22
|
+
* Fedora: `yum install lzo` (or `lzo-devel`)
|
23
|
+
|
24
|
+
Then the bog-standard `gem install lzo` - you know the drill.
|
25
|
+
|
26
|
+
## Usage
|
27
|
+
|
28
|
+
```ruby
|
29
|
+
LZO.decompress(string_or_io) # returns String
|
30
|
+
LZO.compress(string_or_io) # returns String
|
31
|
+
|
32
|
+
file = File.open('/path/to/file.lzo', 'rb')
|
33
|
+
reader = LZO::LzopDecompressor.new file
|
34
|
+
reader.name # => "file.txt"
|
35
|
+
reader.mode # => 0100644
|
36
|
+
reader.mtime # => 2016-02-03 14:29:36 +1100
|
37
|
+
reader.method # => :M_LZO1X_1
|
38
|
+
reader.level # => 5
|
39
|
+
reader.read(10) # => "The quick "
|
40
|
+
|
41
|
+
output = File.open('/path/to/output.lzo', 'wb')
|
42
|
+
writer = LZO::LzopCompressor.new output, name: 'output.txt', mode: 0100644, mtime: Time.now
|
43
|
+
writer.write "first chunk of data"
|
44
|
+
writer.write "second chunk of data"
|
45
|
+
writer.close
|
46
|
+
```
|
data/Rakefile
ADDED
data/TODO.md
ADDED
data/lib/lzo.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'ffi'
|
2
|
+
|
3
|
+
require 'lzo/raw'
|
4
|
+
require 'lzo/ffi'
|
5
|
+
require 'lzo/lzop'
|
6
|
+
require 'lzo/version'
|
7
|
+
require 'lzo/multi_io'
|
8
|
+
|
9
|
+
module LZO
|
10
|
+
class << self
|
11
|
+
def compress(input, lzop: false)
|
12
|
+
io = input.respond_to?(:read) ? input : StringIO.new(input)
|
13
|
+
|
14
|
+
if lzop
|
15
|
+
output_io = StringIO.new ''.force_encoding 'BINARY'
|
16
|
+
compressor = LzopCompressor.new output_io, name: '', mode: 0100644, mtime: Time.now
|
17
|
+
|
18
|
+
until io.eof?
|
19
|
+
bytes = io.read 2**20 # 1MB chunks
|
20
|
+
compressor.write bytes
|
21
|
+
end
|
22
|
+
|
23
|
+
compressor.close
|
24
|
+
output_io.string
|
25
|
+
else
|
26
|
+
Raw.compress io
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def decompress(input)
|
31
|
+
io = input.respond_to?(:read) ? input : StringIO.new(input)
|
32
|
+
|
33
|
+
magic = LZOP::Header::MAGIC
|
34
|
+
start = io.read magic.length
|
35
|
+
multi_io = MultiIO.new [StringIO.new(start), io]
|
36
|
+
|
37
|
+
if start == magic
|
38
|
+
LzopDecompressor.new(multi_io).read
|
39
|
+
else
|
40
|
+
Raw.decompress multi_io
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def library_version
|
45
|
+
FFI.lzo_version
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/lzo/ffi.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
module LZO
|
2
|
+
module FFI
|
3
|
+
extend ::FFI::Library
|
4
|
+
ffi_lib 'lzo2'
|
5
|
+
|
6
|
+
# TODO: all "uint64" refs should check platform bitness
|
7
|
+
WORKMEM_SIZE_32 = 65_536
|
8
|
+
WORKMEM_SIZE_64 = 131_072
|
9
|
+
|
10
|
+
enum :lzo_error_code, [
|
11
|
+
:LZO_E_INTERNAL_ERROR, -99,
|
12
|
+
:LZO_E_OUTPUT_NOT_CONSUMED, -12,
|
13
|
+
:LZO_E_INVALID_ALIGNMENT,
|
14
|
+
:LZO_E_INVALID_ARGUMENT,
|
15
|
+
:LZO_E_NOT_YET_IMPLEMENTED,
|
16
|
+
:LZO_E_INPUT_NOT_CONSUMED,
|
17
|
+
:LZO_E_EOF_NOT_FOUND,
|
18
|
+
:LZO_E_LOOKBEHIND_OVERRUN,
|
19
|
+
:LZO_E_OUTPUT_OVERRUN,
|
20
|
+
:LZO_E_INPUT_OVERRUN,
|
21
|
+
:LZO_E_NOT_COMPRESSIBLE,
|
22
|
+
:LZO_E_OUT_OF_MEMORY,
|
23
|
+
:LZO_E_ERROR,
|
24
|
+
:LZO_E_OK
|
25
|
+
]
|
26
|
+
|
27
|
+
attach_function :lzo1x_1_compress, [
|
28
|
+
:buffer_in, # input buffer
|
29
|
+
:uint, # input buffer length
|
30
|
+
:buffer_out, # output buffer
|
31
|
+
:pointer, # pointer to output buffer length
|
32
|
+
:buffer_inout # work mem
|
33
|
+
], :lzo_error_code
|
34
|
+
|
35
|
+
attach_function :lzo1x_decompress_safe, [
|
36
|
+
:buffer_in, # input buffer
|
37
|
+
:uint, # input buffer length
|
38
|
+
:buffer_out, # output buffer
|
39
|
+
:pointer, # pointer to output buffer length
|
40
|
+
:pointer # work mem (always NULL)
|
41
|
+
], :lzo_error_code
|
42
|
+
|
43
|
+
attach_function :lzo_adler32, [:uint, :buffer_in, :uint], :uint
|
44
|
+
attach_function :lzo_crc32, [:uint, :buffer_in, :uint], :uint
|
45
|
+
attach_function :lzo_version, [], :uint
|
46
|
+
end
|
47
|
+
end
|
data/lib/lzo/lzop.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'bindata'
|
2
|
+
|
3
|
+
require 'lzo/lzop/lzop_decompressor'
|
4
|
+
require 'lzo/lzop/lzop_compressor'
|
5
|
+
require 'lzo/lzop/header_flags'
|
6
|
+
require 'lzo/lzop/lzo_method'
|
7
|
+
require 'lzo/lzop/header'
|
8
|
+
require 'lzo/lzop/block'
|
9
|
+
|
10
|
+
module LZO
|
11
|
+
module LZOP
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module LZO
|
2
|
+
module LZOP
|
3
|
+
class Block < BinData::Record
|
4
|
+
mandatory_parameter :flags
|
5
|
+
hide :block
|
6
|
+
endian :big
|
7
|
+
virtual :flags_ary, value: :flags # TODO: is this necessary?
|
8
|
+
|
9
|
+
uint32 :decompressed_length, assert: -> { value > 0 }
|
10
|
+
uint32 :compressed_length, value: -> { block.length }
|
11
|
+
uint32 :decompressed_checksum, onlyif: -> { flags.include?(:F_ADLER32_D) || flags.include?(:F_CRC32_D) }
|
12
|
+
uint32 :compressed_checksum, onlyif: -> { flags.include?(:F_ADLER32_C) || flags.include?(:F_CRC32_C) }
|
13
|
+
string :block, read_length: :compressed_length
|
14
|
+
|
15
|
+
def body(validate: true)
|
16
|
+
# TODO: spec for raising error when data is invalid
|
17
|
+
raise 'Invalid data' unless !validate || valid?
|
18
|
+
str = block.force_encoding 'BINARY'
|
19
|
+
|
20
|
+
if compressed?
|
21
|
+
io = StringIO.new str
|
22
|
+
Raw.decompress io
|
23
|
+
else
|
24
|
+
str
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def compressed?
|
29
|
+
compressed_length.to_i < decompressed_length.to_i
|
30
|
+
end
|
31
|
+
|
32
|
+
def valid?
|
33
|
+
fa = flags_ary
|
34
|
+
is_adler = fa.include?(:F_ADLER32_D) || fa.include?(:F_ADLER32_C)
|
35
|
+
is_compressed = fa.include?(:F_ADLER32_C) || fa.include?(:F_CRC32_C)
|
36
|
+
|
37
|
+
checksum_method = is_adler ? Raw.method(:adler32) : Raw.method(:crc32)
|
38
|
+
checksum_body = is_compressed ? block.to_s : body(validate: false)
|
39
|
+
|
40
|
+
expected_checksum = is_compressed ? compressed_checksum.to_i : decompressed_checksum.to_i
|
41
|
+
calculated_checksum = checksum_method.call(checksum_body)
|
42
|
+
|
43
|
+
calculated_checksum == expected_checksum
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module LZO
|
2
|
+
module LZOP
|
3
|
+
class Header < BinData::Record
|
4
|
+
MAGIC = "\x89\x4C\x5A\x4F\x00\x0D\x0A\x1A\x0A".force_encoding('BINARY')
|
5
|
+
endian :big
|
6
|
+
|
7
|
+
string :magic, read_length: 9, asserted_value: MAGIC
|
8
|
+
|
9
|
+
struct :checksummable do
|
10
|
+
uint16 :version
|
11
|
+
uint16 :lib_version
|
12
|
+
uint16 :version_needed, onlyif: -> { version >= 0x0940 }
|
13
|
+
LzoMethod :lzo_method
|
14
|
+
uint8 :level, onlyif: -> { version >= 0x940 }
|
15
|
+
HeaderFlags :flags
|
16
|
+
uint32 :filter, onlyif: -> { flags.include? :F_H_FILTER }
|
17
|
+
uint32 :mode
|
18
|
+
uint32 :mtime_low
|
19
|
+
uint32 :mtime_high, onlyif: -> { version >= 0x0940 }
|
20
|
+
uint8 :name_length, value: -> { name.length }
|
21
|
+
string :name, read_length: :name_length
|
22
|
+
end
|
23
|
+
|
24
|
+
uint32 :checksum, asserted_value: -> { Raw.adler32(checksummable.to_binary_s) }
|
25
|
+
struct :extra, onlyif: -> { checksummable.flags.include? :F_H_EXTRA_FIELD } do
|
26
|
+
uint32 :extra_length
|
27
|
+
string :extra_field, length: :extra_length
|
28
|
+
uint32 :extra_checksum
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module LZO
|
2
|
+
module LZOP
|
3
|
+
class HeaderFlags < BinData::Primitive
|
4
|
+
uint32be :flags
|
5
|
+
|
6
|
+
# rubocop:disable Metrics/MethodLength
|
7
|
+
def map
|
8
|
+
{
|
9
|
+
(1 << 0) => :F_ADLER32_D,
|
10
|
+
(1 << 1) => :F_ADLER32_C,
|
11
|
+
(1 << 2) => :F_STDIN,
|
12
|
+
(1 << 3) => :F_STDOUT,
|
13
|
+
(1 << 4) => :F_NAME_DEFAULT,
|
14
|
+
(1 << 5) => :F_DOSISH,
|
15
|
+
(1 << 6) => :F_H_EXTRA_FIELD,
|
16
|
+
(1 << 7) => :F_H_GMTDIFF,
|
17
|
+
(1 << 8) => :F_CRC32_D,
|
18
|
+
(1 << 9) => :F_CRC32_C,
|
19
|
+
(1 << 10) => :F_MULTIPART,
|
20
|
+
(1 << 11) => :F_H_FILTER,
|
21
|
+
(1 << 12) => :F_H_CRC32,
|
22
|
+
(1 << 13) => :F_H_PATH
|
23
|
+
}
|
24
|
+
end
|
25
|
+
# rubocop:enable Metrics/MethodLength
|
26
|
+
|
27
|
+
def get
|
28
|
+
ary = [flags.to_i]
|
29
|
+
map.each do |k, v|
|
30
|
+
ary << v if k & flags.to_i > 0
|
31
|
+
end
|
32
|
+
ary
|
33
|
+
end
|
34
|
+
|
35
|
+
def set(v)
|
36
|
+
tmp = 0
|
37
|
+
v.each do |f|
|
38
|
+
if f.is_a?(Symbol)
|
39
|
+
tmp |= map.invert[f]
|
40
|
+
else
|
41
|
+
tmp |= f
|
42
|
+
end
|
43
|
+
end
|
44
|
+
self.flags = tmp
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module LZO
|
2
|
+
module LZOP
|
3
|
+
class LzoMethod < BinData::Primitive
|
4
|
+
uint8be :val
|
5
|
+
|
6
|
+
def map
|
7
|
+
{ M_LZO1X_1: 1, M_LZO1X_1_15: 2, M_LZO1X_999: 3 }
|
8
|
+
end
|
9
|
+
|
10
|
+
def get
|
11
|
+
map.invert[val.to_i]
|
12
|
+
end
|
13
|
+
|
14
|
+
def set(v)
|
15
|
+
self.val = map[v]
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module LZO
|
2
|
+
class LzopCompressor
|
3
|
+
def initialize(io, name:, mode:, mtime:)
|
4
|
+
@io = io
|
5
|
+
@name = name
|
6
|
+
@mode = mode
|
7
|
+
@mtime = mtime
|
8
|
+
end
|
9
|
+
|
10
|
+
def write(bytes)
|
11
|
+
write_header_once!
|
12
|
+
|
13
|
+
block = LZOP::Block.new flags: [:F_ADLER32_D]
|
14
|
+
|
15
|
+
checksum = Raw.adler32 bytes
|
16
|
+
compressed = Raw.compress StringIO.new bytes
|
17
|
+
|
18
|
+
if compressed.length >= bytes.length
|
19
|
+
block.block = bytes
|
20
|
+
else
|
21
|
+
block.block = compressed
|
22
|
+
end
|
23
|
+
|
24
|
+
# NOTE: compressed_length is set by bindata
|
25
|
+
block.decompressed_checksum = checksum
|
26
|
+
block.decompressed_length = bytes.length
|
27
|
+
|
28
|
+
io.write block.to_binary_s
|
29
|
+
end
|
30
|
+
|
31
|
+
def close
|
32
|
+
eof_marker = [0].pack 'N'
|
33
|
+
io.write eof_marker
|
34
|
+
io.close
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
attr_reader :io, :name, :mode, :mtime
|
39
|
+
|
40
|
+
def write_header_once!
|
41
|
+
@header ||= begin
|
42
|
+
header = LZOP::Header.new
|
43
|
+
fields = header.checksummable
|
44
|
+
fields.version = 0x0940
|
45
|
+
fields.lib_version = LZO.library_version
|
46
|
+
fields.version_needed = 0x0940
|
47
|
+
fields.lzo_method = :M_LZO1X_1
|
48
|
+
fields.level = 1
|
49
|
+
fields.flags = [:F_ADLER32_D]
|
50
|
+
fields.mode = mode
|
51
|
+
fields.mtime_low = mtime.to_i
|
52
|
+
fields.name = name
|
53
|
+
io.write header.to_binary_s
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module LZO
|
2
|
+
class LzopDecompressor
|
3
|
+
def initialize(io)
|
4
|
+
@io = io
|
5
|
+
end
|
6
|
+
|
7
|
+
def name
|
8
|
+
header.checksummable.name.to_s
|
9
|
+
end
|
10
|
+
|
11
|
+
def mode
|
12
|
+
header.checksummable.mode.to_i
|
13
|
+
end
|
14
|
+
|
15
|
+
def mtime
|
16
|
+
Time.at header.checksummable.mtime_low
|
17
|
+
end
|
18
|
+
|
19
|
+
def method
|
20
|
+
header.checksummable.lzo_method.get
|
21
|
+
end
|
22
|
+
|
23
|
+
def level
|
24
|
+
header.checksummable.level.to_i
|
25
|
+
end
|
26
|
+
|
27
|
+
def read(nbytes = nil)
|
28
|
+
@enum ||= enum_for :yield_blocks, io, header
|
29
|
+
@leftover ||= ''.force_encoding 'BINARY'
|
30
|
+
|
31
|
+
output = StringIO.new @leftover.dup
|
32
|
+
@leftover = ''.force_encoding 'BINARY'
|
33
|
+
|
34
|
+
output.seek 0, ::IO::SEEK_END
|
35
|
+
|
36
|
+
while nbytes.nil? || output.length < nbytes
|
37
|
+
begin
|
38
|
+
block = @enum.next
|
39
|
+
rescue StopIteration
|
40
|
+
break
|
41
|
+
end
|
42
|
+
|
43
|
+
bytes = block.body
|
44
|
+
output.write bytes
|
45
|
+
end
|
46
|
+
|
47
|
+
if nbytes && nbytes < output.length
|
48
|
+
output.seek nbytes, ::IO::SEEK_SET
|
49
|
+
@leftover = output.read
|
50
|
+
end
|
51
|
+
|
52
|
+
output.rewind
|
53
|
+
output.read nbytes
|
54
|
+
end
|
55
|
+
|
56
|
+
def eof?
|
57
|
+
io.eof? && @leftover.length == 0
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
attr_reader :io
|
63
|
+
|
64
|
+
def header
|
65
|
+
@header ||= LZOP::Header.read io
|
66
|
+
end
|
67
|
+
|
68
|
+
def yield_blocks(io, header)
|
69
|
+
loop do
|
70
|
+
# TODO: there's gotta be a better way than a validity error?
|
71
|
+
begin
|
72
|
+
block = LZOP::Block.read(io, flags: header.checksummable.flags)
|
73
|
+
yield block
|
74
|
+
rescue BinData::ValidityError
|
75
|
+
# this "exception" is expected: last block of the stream
|
76
|
+
break
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
data/lib/lzo/multi_io.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
module LZO
|
2
|
+
# TODO: surely there's a better way to do this that i'm not aware of
|
3
|
+
class MultiIO
|
4
|
+
def initialize(ios)
|
5
|
+
@ios = ios
|
6
|
+
end
|
7
|
+
|
8
|
+
def read(nbytes = nil)
|
9
|
+
output = StringIO.new ''.force_encoding('BINARY')
|
10
|
+
|
11
|
+
while nbytes.nil? || output.length < nbytes
|
12
|
+
io = @ios.first
|
13
|
+
break if io.nil?
|
14
|
+
read_len = (nbytes - output.length) if nbytes
|
15
|
+
bytes = io.read read_len
|
16
|
+
@ios.shift if io.eof?
|
17
|
+
output.write bytes
|
18
|
+
end
|
19
|
+
|
20
|
+
output.rewind
|
21
|
+
output.read
|
22
|
+
end
|
23
|
+
|
24
|
+
def eof?
|
25
|
+
@ios.length == 0
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/lzo/raw.rb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
module LZO
|
2
|
+
module Raw
|
3
|
+
class << self
|
4
|
+
def compress(io)
|
5
|
+
str = ''
|
6
|
+
str << io.read until io.eof?
|
7
|
+
|
8
|
+
buf_len = str.bytesize
|
9
|
+
buf = ::FFI::MemoryPointer.new :char, buf_len
|
10
|
+
buf.put_bytes 0, str
|
11
|
+
|
12
|
+
out_len = output_length_for_input str
|
13
|
+
out_len_buf = ::FFI::MemoryPointer.new :size_t, 1
|
14
|
+
out_len_buf.put_uint64 0, out_len
|
15
|
+
|
16
|
+
out_buf = ::FFI::MemoryPointer.new :char, out_len
|
17
|
+
|
18
|
+
work_mem_buf = ::FFI::MemoryPointer.new :char, FFI::WORKMEM_SIZE_64
|
19
|
+
|
20
|
+
ret_code = FFI.lzo1x_1_compress buf, buf_len, out_buf, out_len_buf, work_mem_buf
|
21
|
+
raise "Unexpected error code: #{ret_code}" if ret_code != :LZO_E_OK # apparently never fails?
|
22
|
+
|
23
|
+
out_len = out_len_buf.read_uint64
|
24
|
+
out_buf.read_bytes out_len
|
25
|
+
end
|
26
|
+
|
27
|
+
def decompress(io)
|
28
|
+
str = ''
|
29
|
+
str << io.read until io.eof?
|
30
|
+
|
31
|
+
buf_len = str.bytesize
|
32
|
+
buf = ::FFI::MemoryPointer.new :char, buf_len
|
33
|
+
buf.put_bytes 0, str
|
34
|
+
|
35
|
+
# TODO: how can we realloc? this is wasteful
|
36
|
+
out_len = 512
|
37
|
+
begin
|
38
|
+
out_len *= 2
|
39
|
+
out_len_buf = ::FFI::MemoryPointer.new :size_t, 1
|
40
|
+
out_len_buf.put_uint64 0, out_len
|
41
|
+
out_buf = ::FFI::MemoryPointer.new :char, out_len
|
42
|
+
|
43
|
+
ret_code = FFI.lzo1x_decompress_safe buf, buf_len, out_buf, out_len_buf, nil
|
44
|
+
end while ret_code == :LZO_E_OUTPUT_OVERRUN
|
45
|
+
|
46
|
+
if ret_code == :LZO_E_OK
|
47
|
+
out_len = out_len_buf.read_uint64
|
48
|
+
out_buf.read_bytes out_len
|
49
|
+
else
|
50
|
+
raise "Error: #{ret_code}" # TODO: do something?
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def output_length_for_input(input)
|
55
|
+
inlen = input.length
|
56
|
+
inlen + (inlen / 16) + 64 + 3 # worst case
|
57
|
+
end
|
58
|
+
|
59
|
+
def adler32(input)
|
60
|
+
buf_len = input.bytesize
|
61
|
+
buf = ::FFI::MemoryPointer.new :char, buf_len
|
62
|
+
buf.put_bytes 0, input
|
63
|
+
|
64
|
+
FFI.lzo_adler32 1, buf, buf_len
|
65
|
+
end
|
66
|
+
|
67
|
+
def crc32(input)
|
68
|
+
buf_len = input.bytesize
|
69
|
+
buf = ::FFI::MemoryPointer.new :char, buf_len
|
70
|
+
buf.put_bytes 0, input
|
71
|
+
|
72
|
+
FFI.lzo_crc32 0, buf, buf_len
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
data/lib/lzo/version.rb
ADDED
data/lzo.gemspec
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'lzo/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'lzo'
|
8
|
+
spec.version = LZO::VERSION
|
9
|
+
spec.authors = ['Aidan Steele']
|
10
|
+
spec.email = ['aidan@awsteele.com']
|
11
|
+
|
12
|
+
spec.summary = 'LZO bindings for all Rubies + LZOP streaming'
|
13
|
+
spec.homepage = 'https://github.com/aidansteele/lzo'
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
|
16
|
+
spec.bindir = 'exe'
|
17
|
+
spec.executables = spec.files.grep(%r{^exe/}) {|f| File.basename(f) }
|
18
|
+
spec.require_paths = ['lib']
|
19
|
+
|
20
|
+
spec.add_dependency 'ffi', '~> 1'
|
21
|
+
spec.add_dependency 'bindata', '~> 2.1'
|
22
|
+
|
23
|
+
spec.add_development_dependency 'bundler', '~> 1.10'
|
24
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
25
|
+
spec.add_development_dependency 'rspec'
|
26
|
+
spec.add_development_dependency 'timecop'
|
27
|
+
spec.add_development_dependency 'simplecov'
|
28
|
+
spec.add_development_dependency 'rspec_junit_formatter'
|
29
|
+
spec.add_development_dependency 'rubocop'
|
30
|
+
spec.add_development_dependency 'pry-rescue'
|
31
|
+
spec.add_development_dependency 'pry-stack_explorer'
|
32
|
+
end
|
metadata
ADDED
@@ -0,0 +1,219 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: lzo
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Aidan Steele
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-03-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: ffi
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bindata
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '2.1'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '2.1'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.10'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.10'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '10.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '10.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rspec
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: timecop
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: simplecov
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rspec_junit_formatter
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: rubocop
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: pry-rescue
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: pry-stack_explorer
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
167
|
+
description:
|
168
|
+
email:
|
169
|
+
- aidan@awsteele.com
|
170
|
+
executables: []
|
171
|
+
extensions: []
|
172
|
+
extra_rdoc_files: []
|
173
|
+
files:
|
174
|
+
- ".gitignore"
|
175
|
+
- ".rspec"
|
176
|
+
- ".rubocop.yml"
|
177
|
+
- ".ruby-version"
|
178
|
+
- ".travis.yml"
|
179
|
+
- Gemfile
|
180
|
+
- README.md
|
181
|
+
- Rakefile
|
182
|
+
- TODO.md
|
183
|
+
- lib/lzo.rb
|
184
|
+
- lib/lzo/ffi.rb
|
185
|
+
- lib/lzo/lzop.rb
|
186
|
+
- lib/lzo/lzop/block.rb
|
187
|
+
- lib/lzo/lzop/header.rb
|
188
|
+
- lib/lzo/lzop/header_flags.rb
|
189
|
+
- lib/lzo/lzop/lzo_method.rb
|
190
|
+
- lib/lzo/lzop/lzop_compressor.rb
|
191
|
+
- lib/lzo/lzop/lzop_decompressor.rb
|
192
|
+
- lib/lzo/multi_io.rb
|
193
|
+
- lib/lzo/raw.rb
|
194
|
+
- lib/lzo/version.rb
|
195
|
+
- lzo.gemspec
|
196
|
+
homepage: https://github.com/aidansteele/lzo
|
197
|
+
licenses: []
|
198
|
+
metadata: {}
|
199
|
+
post_install_message:
|
200
|
+
rdoc_options: []
|
201
|
+
require_paths:
|
202
|
+
- lib
|
203
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
204
|
+
requirements:
|
205
|
+
- - ">="
|
206
|
+
- !ruby/object:Gem::Version
|
207
|
+
version: '0'
|
208
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
209
|
+
requirements:
|
210
|
+
- - ">="
|
211
|
+
- !ruby/object:Gem::Version
|
212
|
+
version: '0'
|
213
|
+
requirements: []
|
214
|
+
rubyforge_project:
|
215
|
+
rubygems_version: 2.5.1
|
216
|
+
signing_key:
|
217
|
+
specification_version: 4
|
218
|
+
summary: LZO bindings for all Rubies + LZOP streaming
|
219
|
+
test_files: []
|