snappy 0.0.12-java → 0.1.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.travis.yml +28 -1
- data/Gemfile +6 -1
- data/README.md +28 -4
- data/Rakefile +1 -0
- data/ext/extconf.rb +21 -24
- data/lib/snappy.rb +3 -1
- data/lib/snappy/hadoop.rb +22 -0
- data/lib/snappy/hadoop/reader.rb +58 -0
- data/lib/snappy/hadoop/writer.rb +51 -0
- data/lib/snappy/reader.rb +11 -7
- data/lib/snappy/shim.rb +30 -0
- data/lib/snappy/version.rb +3 -1
- data/lib/snappy/writer.rb +14 -9
- data/smoke.sh +8 -0
- data/snappy.gemspec +6 -30
- data/test/hadoop/test-snappy-hadoop-reader.rb +103 -0
- data/test/hadoop/test-snappy-hadoop-writer.rb +48 -0
- data/test/test-snappy-hadoop.rb +22 -0
- data/vendor/snappy/CMakeLists.txt +174 -0
- data/vendor/snappy/CONTRIBUTING.md +26 -0
- data/vendor/snappy/COPYING +1 -1
- data/vendor/snappy/NEWS +52 -0
- data/vendor/snappy/{README → README.md} +23 -9
- data/vendor/snappy/cmake/SnappyConfig.cmake +1 -0
- data/vendor/snappy/cmake/config.h.in +62 -0
- data/vendor/snappy/snappy-c.h +3 -3
- data/vendor/snappy/snappy-internal.h +101 -27
- data/vendor/snappy/snappy-sinksource.cc +33 -0
- data/vendor/snappy/snappy-sinksource.h +51 -6
- data/vendor/snappy/snappy-stubs-internal.h +107 -37
- data/vendor/snappy/snappy-stubs-public.h.in +16 -20
- data/vendor/snappy/snappy-test.cc +15 -9
- data/vendor/snappy/snappy-test.h +34 -43
- data/vendor/snappy/snappy.cc +529 -320
- data/vendor/snappy/snappy.h +23 -4
- data/vendor/snappy/snappy_unittest.cc +240 -185
- metadata +27 -74
- data/vendor/snappy/ChangeLog +0 -1916
- data/vendor/snappy/Makefile.am +0 -23
- data/vendor/snappy/autogen.sh +0 -7
- data/vendor/snappy/configure.ac +0 -133
- data/vendor/snappy/m4/gtest.m4 +0 -74
- data/vendor/snappy/testdata/alice29.txt +0 -3609
- data/vendor/snappy/testdata/asyoulik.txt +0 -4122
- data/vendor/snappy/testdata/baddata1.snappy +0 -0
- data/vendor/snappy/testdata/baddata2.snappy +0 -0
- data/vendor/snappy/testdata/baddata3.snappy +0 -0
- data/vendor/snappy/testdata/fireworks.jpeg +0 -0
- data/vendor/snappy/testdata/geo.protodata +0 -0
- data/vendor/snappy/testdata/html +0 -1
- data/vendor/snappy/testdata/html_x_4 +0 -1
- data/vendor/snappy/testdata/kppkn.gtb +0 -0
- data/vendor/snappy/testdata/lcet10.txt +0 -7519
- data/vendor/snappy/testdata/paper-100k.pdf +2 -600
- data/vendor/snappy/testdata/plrabn12.txt +0 -10699
- data/vendor/snappy/testdata/urls.10K +0 -10000
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 40598da197be3dd37207207288f7d07e864db1854075e05083aab32d0f36c2e7
|
4
|
+
data.tar.gz: 4e530705d88db960c3b0bd3474d33670ae0e79d55d0202770557fca817798efb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 27e1b4cc2d86dff4357130204a94c6212edd8999379be5dcd282ecdad59359ecefded8bc6bb122160683e2a005f8f1d8c229cc2e151c29eaaca0f6967bdd4901
|
7
|
+
data.tar.gz: 240cab9f029fdbe3132bfa76f846135b1d38d9661af81e353650a0674d62333a1025b8bbbcb291167ceca704d6ae10487cb78eb47009aac689af93542f5be474
|
data/.travis.yml
CHANGED
@@ -1,4 +1,31 @@
|
|
1
1
|
language: ruby
|
2
|
+
sudo: false
|
3
|
+
cache:
|
4
|
+
bundler: true
|
5
|
+
directories:
|
6
|
+
- "$HOME/.ccache"
|
2
7
|
rvm:
|
3
|
-
-
|
8
|
+
- 1.9.3
|
9
|
+
- 2.2
|
10
|
+
- 2.3
|
11
|
+
- 2.4
|
12
|
+
- 2.5
|
4
13
|
- jruby-19mode
|
14
|
+
env:
|
15
|
+
global:
|
16
|
+
- PATH="/usr/lib/ccache:$PATH"
|
17
|
+
before_install:
|
18
|
+
- travis_retry gem install bundler --no-document -v 1.16.1
|
19
|
+
addons:
|
20
|
+
apt:
|
21
|
+
packages:
|
22
|
+
- cmake
|
23
|
+
deploy:
|
24
|
+
provider: rubygems
|
25
|
+
api_key:
|
26
|
+
secure: Jwk5pSUD9dUb34fO4dLamqhQw9U/t/5pHX7l5LKE4e1MDlD9fq/KB/DFpRTSfA0s8XUu+NZYtRO+6GZBEBY5UjZyPcUk5izaQzMWANP6QEMI4mYheSFUXTGkimV8MFxo2Fp7itZBM9SsLccDp/59nmbhBUMdr4OECZtd1Igjz63AtRdD7JgZwhC2XqBVbcxyx78VcZH8r4JEZzpcnZjjGN9YwQFubhiPs0V2uTMp/kGSWeP4U3T71MrvRmDlu8QJTsTRsvhv+zDatFySzs2SGsn25+cUCOqWnb5vcwtCbfj7trY+DLO3FqZOvQlvDfvyOy9LlLRpovhu6w+L42Z2ZN5U1Ovz0bMjk5WGLlXRxCLTRqXihvJrtFuL/7B2YSXcWPBNt5HWFwtapEJDm+CJoKpqa/WB7gEgO+uAocHbRqbg64rnPLODOEH0xGeRNSXBA3P5JeVNYhvpFdX8LclRFaVpfYDDenzRV9zpIppMXSq3Suzw/nZPBL9cuuIsBYG6hnijhZertWIRjZqw8D/1wEBd6VT7cDQ+PsNBGKhvPBVd5Nb7Zh9YjwCv4BQHB6XekiKi7mGFN4X/6eTzudRYxG7KyrYqk9AQ6KmEv5U/Xv+Wcar86O49LtNqYykYKLknC9CiUwTDp4lkVD8D3LoIzkL2/AeZyI6NQso8tymB0CM=
|
27
|
+
gem: snappy
|
28
|
+
on:
|
29
|
+
tags: true
|
30
|
+
repo: miyucy/snappy
|
31
|
+
script: bundle exec rake && ./smoke.sh
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -4,13 +4,27 @@ see https://github.com/google/snappy
|
|
4
4
|
|
5
5
|
## Preparation
|
6
6
|
|
7
|
-
Use libsnappy
|
8
7
|
|
9
|
-
|
8
|
+
### macOS
|
10
9
|
|
11
|
-
|
10
|
+
```bash
|
11
|
+
$ brew install snappy
|
12
|
+
$ brew install autoconf automake libtool
|
13
|
+
```
|
14
|
+
|
15
|
+
### Ubuntu
|
16
|
+
|
17
|
+
```bash
|
18
|
+
$ apt-get install libsnappy-dev -y
|
19
|
+
$ apt-get install libtool automake autoconf -y
|
20
|
+
```
|
21
|
+
|
22
|
+
### Alpine
|
12
23
|
|
13
|
-
|
24
|
+
```bash
|
25
|
+
$ apk install snappy
|
26
|
+
$ apk install build-base libexecinfo automake autoconf libtool
|
27
|
+
```
|
14
28
|
|
15
29
|
## Installation
|
16
30
|
|
@@ -28,6 +42,7 @@ Or install it yourself as:
|
|
28
42
|
|
29
43
|
## Usage
|
30
44
|
|
45
|
+
Snappy-java format
|
31
46
|
```ruby
|
32
47
|
require 'snappy'
|
33
48
|
|
@@ -38,6 +53,15 @@ Snappy.inflate(source)
|
|
38
53
|
# => Decompressed data
|
39
54
|
```
|
40
55
|
|
56
|
+
Hadoop-snappy format
|
57
|
+
```ruby
|
58
|
+
Snappy::Hadoop.deflate(source)
|
59
|
+
# => Compressed data
|
60
|
+
|
61
|
+
Snappy::Hadoop.inflate(source)
|
62
|
+
# => Decompressed data
|
63
|
+
```
|
64
|
+
|
41
65
|
## Contributing
|
42
66
|
|
43
67
|
1. Fork it
|
data/Rakefile
CHANGED
data/ext/extconf.rb
CHANGED
@@ -1,37 +1,34 @@
|
|
1
1
|
require 'mkmf'
|
2
2
|
require 'fileutils'
|
3
3
|
|
4
|
-
|
5
|
-
# s/libtoolize/glibtoolize/
|
6
|
-
File.write('autogen.sh', File.read('autogen.sh').gsub(/libtoolize/, 'glibtoolize'))
|
7
|
-
end
|
8
|
-
|
9
|
-
unless have_library 'snappy'
|
4
|
+
unless pkg_config('libsnappy') || have_library('snappy')
|
10
5
|
# build vendor/snappy
|
11
6
|
pwd = File.dirname File.expand_path __FILE__
|
12
7
|
dir = File.join pwd, '..', 'vendor', 'snappy'
|
13
8
|
|
14
|
-
Dir.chdir
|
15
|
-
|
16
|
-
|
17
|
-
|
9
|
+
Dir.chdir(dir) do
|
10
|
+
FileUtils.mkdir_p 'build'
|
11
|
+
Dir.chdir(File.join(dir, 'build')) do
|
12
|
+
`cmake .. -DCMAKE_BUILD_TYPE=Release`
|
18
13
|
end
|
19
|
-
raise '`configure` failed' unless system './configure --disable-option-checking --disable-dependency-tracking --disable-gtest --without-gflags'
|
20
14
|
end
|
21
15
|
|
22
|
-
src = %w
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
16
|
+
src = %w[
|
17
|
+
config.h
|
18
|
+
snappy-stubs-public.h
|
19
|
+
].map { |e| File.join dir, 'build', e }
|
20
|
+
FileUtils.cp src, pwd, :verbose => true
|
21
|
+
src = %w[
|
22
|
+
snappy-c.cc
|
23
|
+
snappy-c.h
|
24
|
+
snappy-internal.h
|
25
|
+
snappy-sinksource.cc
|
26
|
+
snappy-sinksource.h
|
27
|
+
snappy.cc
|
28
|
+
snappy.h
|
29
|
+
snappy-stubs-internal.cc
|
30
|
+
snappy-stubs-internal.h
|
31
|
+
].map { |e| File.join dir, e }
|
35
32
|
FileUtils.cp src, pwd, :verbose => true
|
36
33
|
end
|
37
34
|
|
data/lib/snappy.rb
CHANGED
@@ -0,0 +1,22 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'stringio'
|
5
|
+
require 'snappy/hadoop/reader'
|
6
|
+
require 'snappy/hadoop/writer'
|
7
|
+
|
8
|
+
module Snappy
|
9
|
+
module Hadoop
|
10
|
+
def self.deflate(source, block_size = Snappy::Hadoop::Writer::DEFAULT_BLOCK_SIZE)
|
11
|
+
compressed_io = StringIO.new
|
12
|
+
writer = Snappy::Hadoop::Writer.new(compressed_io)
|
13
|
+
writer << source
|
14
|
+
writer.flush
|
15
|
+
compressed_io.string
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.inflate(source)
|
19
|
+
Snappy::Hadoop::Reader.new(StringIO.new(source)).read
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'stringio'
|
4
|
+
require 'snappy/shim'
|
5
|
+
|
6
|
+
module Snappy
|
7
|
+
module Hadoop
|
8
|
+
class Reader
|
9
|
+
attr_reader :io
|
10
|
+
|
11
|
+
def initialize(io)
|
12
|
+
@io = Snappy.set_encoding io
|
13
|
+
yield self if block_given?
|
14
|
+
end
|
15
|
+
|
16
|
+
def each
|
17
|
+
until @io.eof?
|
18
|
+
# Uncompressed size (32 bit integer, BE).
|
19
|
+
uncompressed_size = @io.read(4).unpack('N').first
|
20
|
+
|
21
|
+
uncompressed_block_io = StringIO.new
|
22
|
+
while uncompressed_block_io.size < uncompressed_size
|
23
|
+
# Compressed subblock size (32 bit integer, BE).
|
24
|
+
compressed_size = @io.read(4).unpack('N').first
|
25
|
+
uncompressed_block_io << Snappy.inflate(@io.read(compressed_size))
|
26
|
+
end
|
27
|
+
|
28
|
+
if uncompressed_block_io.size > uncompressed_size
|
29
|
+
raise RuntimeError("Invalid data: expected #{uncompressed_size} bytes, got #{Uncompressed.size}")
|
30
|
+
end
|
31
|
+
|
32
|
+
yield uncompressed_block_io.string if block_given?
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def read
|
37
|
+
buff = StringIO.new
|
38
|
+
each do |chunk|
|
39
|
+
buff << chunk
|
40
|
+
end
|
41
|
+
buff.string
|
42
|
+
end
|
43
|
+
|
44
|
+
def each_line(sep_string = $/)
|
45
|
+
last = ""
|
46
|
+
each do |chunk|
|
47
|
+
chunk = last + chunk
|
48
|
+
lines = chunk.split(sep_string)
|
49
|
+
last = lines.pop
|
50
|
+
lines.each do |line|
|
51
|
+
yield line if block_given?
|
52
|
+
end
|
53
|
+
end
|
54
|
+
yield last
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'snappy/shim'
|
4
|
+
|
5
|
+
module Snappy
|
6
|
+
module Hadoop
|
7
|
+
class Writer
|
8
|
+
DEFAULT_BLOCK_SIZE = 256 * 1024
|
9
|
+
|
10
|
+
attr_reader :io, :block_size
|
11
|
+
|
12
|
+
def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
|
13
|
+
@block_size = block_size
|
14
|
+
@buffer = String.new
|
15
|
+
@io = Snappy.set_encoding io
|
16
|
+
if block_given?
|
17
|
+
yield self
|
18
|
+
dump!
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def <<(msg)
|
23
|
+
@buffer << msg.to_s
|
24
|
+
dump! if @buffer.size >= @block_size
|
25
|
+
end
|
26
|
+
|
27
|
+
alias_method :write, :<<
|
28
|
+
|
29
|
+
def dump!
|
30
|
+
offset = 0
|
31
|
+
while offset < @buffer.size
|
32
|
+
uncompressed = @buffer[offset, @block_size]
|
33
|
+
compressed = Snappy.deflate(uncompressed)
|
34
|
+
|
35
|
+
# Uncompressed size (32 bit integer, BE), compressed size (32 bit integer, BE), data.
|
36
|
+
@io << [uncompressed.size, compressed.size, compressed].pack("NNa#{compressed.size}")
|
37
|
+
offset += uncompressed.size
|
38
|
+
end
|
39
|
+
|
40
|
+
@io.flush
|
41
|
+
@buffer.clear
|
42
|
+
end
|
43
|
+
|
44
|
+
alias_method :flush, :dump!
|
45
|
+
|
46
|
+
def close
|
47
|
+
@io.close
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
data/lib/snappy/reader.rb
CHANGED
@@ -1,10 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'snappy/shim'
|
4
|
+
require 'snappy/writer'
|
5
|
+
|
1
6
|
module Snappy
|
2
7
|
class Reader
|
3
8
|
attr_reader :io, :magic, :default_version, :minimum_compatible_version
|
4
9
|
|
5
10
|
def initialize(io)
|
6
|
-
@io = io
|
7
|
-
@io.set_encoding Encoding::ASCII_8BIT unless RUBY_VERSION =~ /^1\.8/
|
11
|
+
@io = Snappy.set_encoding io
|
8
12
|
read_header!
|
9
13
|
yield self if block_given?
|
10
14
|
end
|
@@ -15,20 +19,20 @@ module Snappy
|
|
15
19
|
size = @io.read(4).unpack('N').first
|
16
20
|
yield Snappy.inflate(@io.read(size)) if block_given?
|
17
21
|
else
|
18
|
-
yield Snappy.inflate
|
22
|
+
yield Snappy.inflate(@io.read) if block_given?
|
19
23
|
end
|
20
24
|
end
|
21
25
|
end
|
22
26
|
|
23
27
|
def read
|
24
|
-
|
28
|
+
buff = StringIO.new
|
25
29
|
each do |chunk|
|
26
|
-
|
30
|
+
buff << chunk
|
27
31
|
end
|
28
|
-
|
32
|
+
buff.string
|
29
33
|
end
|
30
34
|
|
31
|
-
def each_line(sep_string
|
35
|
+
def each_line(sep_string = $/)
|
32
36
|
last = ""
|
33
37
|
each do |chunk|
|
34
38
|
chunk = last + chunk
|
data/lib/snappy/shim.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Snappy
|
4
|
+
module_function
|
5
|
+
|
6
|
+
if RUBY_VERSION[0..2] == '1.8'
|
7
|
+
def set_encoding(io)
|
8
|
+
io
|
9
|
+
end
|
10
|
+
|
11
|
+
def b(str)
|
12
|
+
str
|
13
|
+
end
|
14
|
+
else
|
15
|
+
def set_encoding(io)
|
16
|
+
io.set_encoding Encoding::ASCII_8BIT
|
17
|
+
io
|
18
|
+
end
|
19
|
+
|
20
|
+
if ::String.instance_methods.include? :b
|
21
|
+
def b(str)
|
22
|
+
str.b
|
23
|
+
end
|
24
|
+
else
|
25
|
+
def b(str)
|
26
|
+
str.force_encoding Encoding::ASCII_8BIT
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/snappy/version.rb
CHANGED
data/lib/snappy/writer.rb
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'snappy/shim'
|
4
|
+
|
1
5
|
module Snappy
|
2
6
|
class Writer
|
3
|
-
|
4
|
-
MAGIC = "\x82SNAPPY\x0"
|
5
|
-
else
|
6
|
-
MAGIC = "\x82SNAPPY\x0".force_encoding Encoding::ASCII_8BIT
|
7
|
-
end
|
7
|
+
MAGIC = Snappy.b("\x82SNAPPY\x0")
|
8
8
|
DEFAULT_VERSION = 1
|
9
9
|
MINIMUM_COMPATIBLE_VERSION = 1
|
10
10
|
DEFAULT_BLOCK_SIZE = 32 * 1024
|
@@ -13,9 +13,8 @@ module Snappy
|
|
13
13
|
|
14
14
|
def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
|
15
15
|
@block_size = block_size
|
16
|
-
@buffer =
|
17
|
-
@io = io
|
18
|
-
@io.set_encoding Encoding::ASCII_8BIT unless RUBY_VERSION =~ /^1\.8/
|
16
|
+
@buffer = String.new
|
17
|
+
@io = Snappy.set_encoding io
|
19
18
|
write_header!
|
20
19
|
if block_given?
|
21
20
|
yield self
|
@@ -28,15 +27,21 @@ module Snappy
|
|
28
27
|
dump! if @buffer.size > @block_size
|
29
28
|
end
|
30
29
|
|
30
|
+
alias_method :write, :<<
|
31
|
+
|
31
32
|
def dump!
|
32
33
|
compressed = Snappy.deflate(@buffer)
|
33
34
|
@io << [compressed.size, compressed].pack("Na#{compressed.size}")
|
34
35
|
@io.flush
|
35
|
-
@buffer
|
36
|
+
@buffer.clear
|
36
37
|
end
|
37
38
|
|
38
39
|
alias_method :flush, :dump!
|
39
40
|
|
41
|
+
def close
|
42
|
+
@io.close
|
43
|
+
end
|
44
|
+
|
40
45
|
private
|
41
46
|
|
42
47
|
def write_header!
|