snappy 0.0.17 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.dockerignore +2 -0
- data/.github/workflows/main.yml +34 -0
- data/.github/workflows/publish.yml +34 -0
- data/.gitignore +2 -1
- data/.gitmodules +1 -1
- data/Dockerfile +13 -0
- data/Gemfile +4 -0
- data/README.md +29 -5
- data/Rakefile +32 -29
- data/ext/api.c +6 -1
- data/ext/extconf.rb +23 -16
- data/lib/snappy/hadoop/reader.rb +62 -0
- data/lib/snappy/hadoop/writer.rb +51 -0
- data/lib/snappy/hadoop.rb +22 -0
- data/lib/snappy/reader.rb +14 -10
- data/lib/snappy/shim.rb +1 -1
- data/lib/snappy/version.rb +1 -1
- data/lib/snappy.rb +5 -4
- data/snappy.gemspec +13 -13
- data/test/hadoop/snappy_hadoop_reader_test.rb +115 -0
- data/test/hadoop/snappy_hadoop_writer_test.rb +48 -0
- data/test/snappy_hadoop_test.rb +26 -0
- data/test/snappy_reader_test.rb +148 -0
- data/test/snappy_test.rb +95 -0
- data/test/snappy_writer_test.rb +55 -0
- data/test/test_helper.rb +7 -0
- data/test.sh +3 -0
- data/vendor/snappy/CMakeLists.txt +297 -0
- data/vendor/snappy/CONTRIBUTING.md +26 -0
- data/vendor/snappy/NEWS +40 -0
- data/vendor/snappy/{README → README.md} +27 -18
- data/vendor/snappy/cmake/SnappyConfig.cmake.in +33 -0
- data/vendor/snappy/cmake/config.h.in +62 -0
- data/vendor/snappy/docs/README.md +72 -0
- data/vendor/snappy/snappy-internal.h +22 -18
- data/vendor/snappy/snappy-stubs-internal.cc +1 -1
- data/vendor/snappy/snappy-stubs-internal.h +116 -38
- data/vendor/snappy/snappy-stubs-public.h.in +20 -46
- data/vendor/snappy/snappy-test.cc +26 -22
- data/vendor/snappy/snappy-test.h +24 -98
- data/vendor/snappy/snappy.cc +380 -183
- data/vendor/snappy/snappy.h +14 -10
- data/vendor/snappy/snappy_compress_fuzzer.cc +59 -0
- data/vendor/snappy/snappy_uncompress_fuzzer.cc +57 -0
- data/vendor/snappy/snappy_unittest.cc +236 -261
- metadata +37 -92
- data/.travis.yml +0 -26
- data/smoke.sh +0 -8
- data/test/test-snappy-reader.rb +0 -129
- data/test/test-snappy-writer.rb +0 -55
- data/test/test-snappy.rb +0 -58
- data/vendor/snappy/ChangeLog +0 -2468
- data/vendor/snappy/INSTALL +0 -370
- data/vendor/snappy/Makefile +0 -982
- data/vendor/snappy/Makefile.am +0 -26
- data/vendor/snappy/Makefile.in +0 -982
- data/vendor/snappy/aclocal.m4 +0 -9738
- data/vendor/snappy/autogen.sh +0 -12
- data/vendor/snappy/autom4te.cache/output.0 +0 -18856
- data/vendor/snappy/autom4te.cache/output.1 +0 -18852
- data/vendor/snappy/autom4te.cache/requests +0 -297
- data/vendor/snappy/autom4te.cache/traces.0 +0 -2689
- data/vendor/snappy/autom4te.cache/traces.1 +0 -714
- data/vendor/snappy/config.guess +0 -1530
- data/vendor/snappy/config.h +0 -135
- data/vendor/snappy/config.h.in +0 -134
- data/vendor/snappy/config.log +0 -1640
- data/vendor/snappy/config.status +0 -2318
- data/vendor/snappy/config.sub +0 -1773
- data/vendor/snappy/configure +0 -18852
- data/vendor/snappy/configure.ac +0 -134
- data/vendor/snappy/depcomp +0 -688
- data/vendor/snappy/install-sh +0 -527
- data/vendor/snappy/libtool +0 -10246
- data/vendor/snappy/ltmain.sh +0 -9661
- data/vendor/snappy/m4/gtest.m4 +0 -74
- data/vendor/snappy/m4/libtool.m4 +0 -8001
- data/vendor/snappy/m4/ltoptions.m4 +0 -384
- data/vendor/snappy/m4/ltsugar.m4 +0 -123
- data/vendor/snappy/m4/ltversion.m4 +0 -23
- data/vendor/snappy/m4/lt~obsolete.m4 +0 -98
- data/vendor/snappy/missing +0 -331
- data/vendor/snappy/snappy-stubs-public.h +0 -100
- data/vendor/snappy/snappy.pc +0 -10
- data/vendor/snappy/snappy.pc.in +0 -10
- data/vendor/snappy/stamp-h1 +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 4866cfb8c9b7b1e16e5d6c8ba7044d8e003d5a03274575a09d60227bbdf4c603
|
4
|
+
data.tar.gz: dfd8a8d4cdb51bce5a988b55c61940d9717fb95a9f9377f32db93708d43805ce
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1f3f35f369f73dee25075f72dc48693833d084d6bc6844243ef82585f41255bbf6bedaece227b05aa3ecc8bd9f9db2a13c98ea9ee1f9894204a719b9e0e0f57c
|
7
|
+
data.tar.gz: 36af767eded664e7bb25e120e807897d47a6d6c5a86d6f655970030a20636b14143ae8e1df84d9032aa175987b2d97a21c31d1c048f0422f7da00079bce38d23
|
data/.dockerignore
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
name: Ruby
|
2
|
+
|
3
|
+
on: [pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
strategy:
|
8
|
+
fail-fast: false
|
9
|
+
matrix:
|
10
|
+
ruby: [2.5, 2.6, 2.7, 3.0, 3.1, jruby]
|
11
|
+
runs-on: ubuntu-latest
|
12
|
+
steps:
|
13
|
+
- uses: actions/checkout@v2
|
14
|
+
with:
|
15
|
+
submodules: true
|
16
|
+
- name: Set up Ruby
|
17
|
+
uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
bundler-cache: true
|
21
|
+
- name: Run the default task
|
22
|
+
run: |
|
23
|
+
bundle exec rake clobber clean test build
|
24
|
+
gem install --no-document "$(ls pkg/snappy-*.gem)"
|
25
|
+
cat <<EOF | ruby
|
26
|
+
require "snappy"
|
27
|
+
if Snappy.inflate(Snappy.deflate(File.read("README.md"))) == File.read("README.md")
|
28
|
+
puts "OK"
|
29
|
+
exit 0
|
30
|
+
else
|
31
|
+
puts "NG"
|
32
|
+
exit 0
|
33
|
+
end
|
34
|
+
EOF
|
@@ -0,0 +1,34 @@
|
|
1
|
+
name: Publish Gem
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
tags:
|
6
|
+
- 'v*'
|
7
|
+
|
8
|
+
jobs:
|
9
|
+
build:
|
10
|
+
strategy:
|
11
|
+
fail-fast: false
|
12
|
+
matrix:
|
13
|
+
ruby: [2.7, jruby]
|
14
|
+
runs-on: ubuntu-latest
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
with:
|
18
|
+
submodules: true
|
19
|
+
- name: Set up Ruby
|
20
|
+
uses: ruby/setup-ruby@v1
|
21
|
+
with:
|
22
|
+
ruby-version: ${{ matrix.ruby }}
|
23
|
+
bundler-cache: true
|
24
|
+
- name: Run release task
|
25
|
+
run: |
|
26
|
+
mkdir -p ~/.gem
|
27
|
+
cat << EOF > ~/.gem/credentials
|
28
|
+
---
|
29
|
+
:github: Bearer ${{secrets.GITHUB_TOKEN}}
|
30
|
+
:rubygems_api_key: ${{secrets.RUBYGEMS_API_KEY}}
|
31
|
+
EOF
|
32
|
+
chmod 600 ~/.gem/credentials
|
33
|
+
bundle exec rake release[remote]
|
34
|
+
rm -f ~/.gem/credentials
|
data/.gitignore
CHANGED
data/.gitmodules
CHANGED
data/Dockerfile
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
# syntax = docker/dockerfile:labs
|
2
|
+
ARG RUBY_VERSION=3.1
|
3
|
+
FROM ruby:${RUBY_VERSION}
|
4
|
+
|
5
|
+
ENV BUNDLE_JOBS=4
|
6
|
+
|
7
|
+
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt <<EOF
|
8
|
+
apt-get update
|
9
|
+
apt-get install -y \
|
10
|
+
cmake
|
11
|
+
EOF
|
12
|
+
|
13
|
+
WORKDIR /app
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,16 +1,30 @@
|
|
1
|
-
# Snappy
|
1
|
+
# Snappy
|
2
2
|
|
3
3
|
see https://github.com/google/snappy
|
4
4
|
|
5
5
|
## Preparation
|
6
6
|
|
7
|
-
Use libsnappy
|
8
7
|
|
9
|
-
|
8
|
+
### macOS
|
10
9
|
|
11
|
-
|
10
|
+
```bash
|
11
|
+
$ brew install snappy
|
12
|
+
$ brew install autoconf automake libtool
|
13
|
+
```
|
14
|
+
|
15
|
+
### Ubuntu
|
16
|
+
|
17
|
+
```bash
|
18
|
+
$ apt-get install libsnappy-dev -y
|
19
|
+
$ apt-get install libtool automake autoconf -y
|
20
|
+
```
|
21
|
+
|
22
|
+
### Alpine
|
12
23
|
|
13
|
-
|
24
|
+
```bash
|
25
|
+
$ apk install snappy
|
26
|
+
$ apk install build-base libexecinfo automake autoconf libtool
|
27
|
+
```
|
14
28
|
|
15
29
|
## Installation
|
16
30
|
|
@@ -28,6 +42,7 @@ Or install it yourself as:
|
|
28
42
|
|
29
43
|
## Usage
|
30
44
|
|
45
|
+
Snappy-java format
|
31
46
|
```ruby
|
32
47
|
require 'snappy'
|
33
48
|
|
@@ -38,6 +53,15 @@ Snappy.inflate(source)
|
|
38
53
|
# => Decompressed data
|
39
54
|
```
|
40
55
|
|
56
|
+
Hadoop-snappy format
|
57
|
+
```ruby
|
58
|
+
Snappy::Hadoop.deflate(source)
|
59
|
+
# => Compressed data
|
60
|
+
|
61
|
+
Snappy::Hadoop.inflate(source)
|
62
|
+
# => Decompressed data
|
63
|
+
```
|
64
|
+
|
41
65
|
## Contributing
|
42
66
|
|
43
67
|
1. Fork it
|
data/Rakefile
CHANGED
@@ -2,48 +2,51 @@ require "bundler/setup"
|
|
2
2
|
require "bundler/gem_tasks"
|
3
3
|
require "rake/testtask"
|
4
4
|
require "rbconfig"
|
5
|
-
DLEXT = RbConfig::CONFIG[
|
5
|
+
DLEXT = RbConfig::CONFIG["DLEXT"]
|
6
6
|
|
7
|
-
Rake::TestTask.new do |t|
|
7
|
+
Rake::TestTask.new(:test) do |t|
|
8
|
+
t.libs << "test"
|
9
|
+
t.libs << "lib"
|
10
|
+
t.test_files = FileList["test/**/*_test.rb"]
|
8
11
|
t.warning = true
|
9
12
|
t.verbose = true
|
10
13
|
end
|
11
14
|
|
12
15
|
if defined?(JRUBY_VERSION)
|
13
|
-
require
|
16
|
+
require "ant"
|
14
17
|
|
15
|
-
directory
|
18
|
+
directory "ext/java/build"
|
16
19
|
|
17
|
-
task :
|
18
|
-
ant.property name:
|
19
|
-
ant.property name:
|
20
|
+
task setup: "ext/java/build" do
|
21
|
+
ant.property name: "src.dir", value: "ext/java/src"
|
22
|
+
ant.property name: "build.dir", value: "ext/java/build"
|
20
23
|
|
21
|
-
ant.path id:
|
22
|
-
pathelement location: File.join(RbConfig::CONFIG[
|
23
|
-
$LOAD_PATH.flat_map { |path| Dir[File.join(path,
|
24
|
+
ant.path id: "compile.class.path" do
|
25
|
+
pathelement location: File.join(RbConfig::CONFIG["prefix"], "lib", "jruby.jar")
|
26
|
+
$LOAD_PATH.flat_map { |path| Dir[File.join(path, "**", "*.jar")] }.each do |jar|
|
24
27
|
pathelement location: jar
|
25
28
|
end
|
26
29
|
end
|
27
30
|
end
|
28
31
|
|
29
|
-
desc
|
30
|
-
task :
|
31
|
-
ant.javac destdir:
|
32
|
-
classpath refid:
|
33
|
-
src { pathelement location:
|
32
|
+
desc "Compile the extension"
|
33
|
+
task compile: :setup do
|
34
|
+
ant.javac destdir: "${build.dir}", includeantruntime: "no", target: "1.6", source: "1.6", debug: "on" do
|
35
|
+
classpath refid: "compile.class.path"
|
36
|
+
src { pathelement location: "${src.dir}" }
|
34
37
|
end
|
35
38
|
end
|
36
39
|
|
37
|
-
desc
|
38
|
-
file
|
39
|
-
ant.jar destfile:
|
40
|
-
ant.fileset dir:
|
41
|
-
ant.fileset dir:
|
40
|
+
desc "Package the jar"
|
41
|
+
file "lib/snappy_ext.jar" => :compile do |t|
|
42
|
+
ant.jar destfile: "lib/snappy_ext.jar", basedir: "${build.dir}" do
|
43
|
+
ant.fileset dir: "${build.dir}", includes: "snappy/*.class"
|
44
|
+
ant.fileset dir: "${build.dir}", includes: "SnappyExtLibraryService.class"
|
42
45
|
end
|
43
46
|
end
|
44
47
|
|
45
|
-
task :
|
46
|
-
task :
|
48
|
+
task test: "lib/snappy_ext.jar"
|
49
|
+
task build: [:clean, "lib/snappy_ext.jar"]
|
47
50
|
else
|
48
51
|
file "ext/snappy_ext.#{DLEXT}" => Dir.glob("ext/*{.rb,.c}") do
|
49
52
|
Dir.chdir("ext") do
|
@@ -53,15 +56,15 @@ else
|
|
53
56
|
cp "ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}"
|
54
57
|
end
|
55
58
|
|
56
|
-
|
57
|
-
task :test => "ext/snappy_ext.#{DLEXT}"
|
59
|
+
task test: "ext/snappy_ext.#{DLEXT}"
|
58
60
|
end
|
59
61
|
|
60
|
-
desc
|
62
|
+
desc "Clean up build artifacts"
|
61
63
|
task :clean do
|
62
|
-
rm_rf
|
63
|
-
rm_rf
|
64
|
-
rm_rf(["ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}",
|
64
|
+
rm_rf "ext/java/build"
|
65
|
+
rm_rf "lib/snappy_ext.jar"
|
66
|
+
rm_rf(["ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}", "ext/mkmf.log", "ext/config.h", "ext/api.o",
|
67
|
+
"ext/Makefile", "ext/snappy.cc", "ext/snappy.h", "ext/snappy.o"] + Dir["ext/snappy-*"])
|
65
68
|
end
|
66
69
|
|
67
|
-
task :
|
70
|
+
task default: :test
|
data/ext/api.c
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
#include "ruby.h"
|
2
2
|
#include "snappy-c.h"
|
3
3
|
|
4
|
-
static VALUE rb_mSnappy;
|
5
4
|
static VALUE rb_eSnappy;
|
6
5
|
|
7
6
|
static VALUE
|
@@ -92,10 +91,16 @@ snappy_valid_p(VALUE self, VALUE str)
|
|
92
91
|
|
93
92
|
void Init_snappy_ext()
|
94
93
|
{
|
94
|
+
#if HAVE_RB_EXT_RACTOR_SAFE
|
95
|
+
rb_ext_ractor_safe(true);
|
96
|
+
#endif
|
97
|
+
|
98
|
+
VALUE rb_mSnappy;
|
95
99
|
VALUE rb_mSnappy_singleton;
|
96
100
|
|
97
101
|
rb_mSnappy = rb_define_module("Snappy");
|
98
102
|
rb_eSnappy = rb_define_class_under(rb_mSnappy, "Error", rb_eStandardError);
|
103
|
+
rb_global_variable(&rb_eSnappy);
|
99
104
|
rb_define_singleton_method(rb_mSnappy, "deflate", snappy_deflate, -1);
|
100
105
|
rb_define_singleton_method(rb_mSnappy, "inflate", snappy_inflate, -1);
|
101
106
|
rb_define_singleton_method(rb_mSnappy, "valid?", snappy_valid_p, 1);
|
data/ext/extconf.rb
CHANGED
@@ -1,29 +1,36 @@
|
|
1
1
|
require 'mkmf'
|
2
2
|
require 'fileutils'
|
3
3
|
|
4
|
+
$CXXFLAGS += " -std=c++11 "
|
5
|
+
|
4
6
|
unless pkg_config('libsnappy') || have_library('snappy')
|
5
7
|
# build vendor/snappy
|
6
8
|
pwd = File.dirname File.expand_path __FILE__
|
7
9
|
dir = File.join pwd, '..', 'vendor', 'snappy'
|
8
10
|
|
9
|
-
Dir.chdir
|
10
|
-
|
11
|
-
|
11
|
+
Dir.chdir(dir) do
|
12
|
+
FileUtils.mkdir_p 'build'
|
13
|
+
Dir.chdir(File.join(dir, 'build')) do
|
14
|
+
`cmake .. -DCMAKE_BUILD_TYPE=Release`
|
15
|
+
end
|
12
16
|
end
|
13
17
|
|
14
|
-
src = %w
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
18
|
+
src = %w[
|
19
|
+
config.h
|
20
|
+
snappy-stubs-public.h
|
21
|
+
].map { |e| File.join dir, 'build', e }
|
22
|
+
FileUtils.cp src, pwd, :verbose => true
|
23
|
+
src = %w[
|
24
|
+
snappy-c.cc
|
25
|
+
snappy-c.h
|
26
|
+
snappy-internal.h
|
27
|
+
snappy-sinksource.cc
|
28
|
+
snappy-sinksource.h
|
29
|
+
snappy.cc
|
30
|
+
snappy.h
|
31
|
+
snappy-stubs-internal.cc
|
32
|
+
snappy-stubs-internal.h
|
33
|
+
].map { |e| File.join dir, e }
|
27
34
|
FileUtils.cp src, pwd, :verbose => true
|
28
35
|
end
|
29
36
|
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'stringio'
|
4
|
+
require 'snappy/shim'
|
5
|
+
|
6
|
+
module Snappy
|
7
|
+
module Hadoop
|
8
|
+
class Reader
|
9
|
+
attr_reader :io
|
10
|
+
|
11
|
+
def initialize(io)
|
12
|
+
@io = Snappy.set_encoding io
|
13
|
+
yield self if block_given?
|
14
|
+
end
|
15
|
+
|
16
|
+
def each
|
17
|
+
return to_enum unless block_given?
|
18
|
+
|
19
|
+
until @io.eof?
|
20
|
+
# Uncompressed size (32 bit integer, BE).
|
21
|
+
uncompressed_size = @io.read(4).unpack('N').first
|
22
|
+
|
23
|
+
uncompressed_block_io = StringIO.new
|
24
|
+
while uncompressed_block_io.size < uncompressed_size
|
25
|
+
# Compressed subblock size (32 bit integer, BE).
|
26
|
+
compressed_size = @io.read(4).unpack('N').first
|
27
|
+
uncompressed_block_io << Snappy.inflate(@io.read(compressed_size))
|
28
|
+
end
|
29
|
+
|
30
|
+
if uncompressed_block_io.size > uncompressed_size
|
31
|
+
raise RuntimeError("Invalid data: expected #{uncompressed_size} bytes, got #{Uncompressed.size}")
|
32
|
+
end
|
33
|
+
|
34
|
+
yield uncompressed_block_io.string
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def read
|
39
|
+
buff = StringIO.new
|
40
|
+
each do |chunk|
|
41
|
+
buff << chunk
|
42
|
+
end
|
43
|
+
buff.string
|
44
|
+
end
|
45
|
+
|
46
|
+
def each_line(sep_string = $/)
|
47
|
+
return to_enum(:each_line, sep_string) unless block_given?
|
48
|
+
|
49
|
+
last = ""
|
50
|
+
each do |chunk|
|
51
|
+
chunk = last + chunk
|
52
|
+
lines = chunk.split(sep_string)
|
53
|
+
last = lines.pop
|
54
|
+
lines.each do |line|
|
55
|
+
yield line
|
56
|
+
end
|
57
|
+
end
|
58
|
+
yield last
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'snappy/shim'
|
4
|
+
|
5
|
+
module Snappy
|
6
|
+
module Hadoop
|
7
|
+
class Writer
|
8
|
+
DEFAULT_BLOCK_SIZE = 256 * 1024
|
9
|
+
|
10
|
+
attr_reader :io, :block_size
|
11
|
+
|
12
|
+
def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
|
13
|
+
@block_size = block_size
|
14
|
+
@buffer = String.new
|
15
|
+
@io = Snappy.set_encoding io
|
16
|
+
if block_given?
|
17
|
+
yield self
|
18
|
+
dump!
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def <<(msg)
|
23
|
+
@buffer << msg.to_s
|
24
|
+
dump! if @buffer.size >= @block_size
|
25
|
+
end
|
26
|
+
|
27
|
+
alias_method :write, :<<
|
28
|
+
|
29
|
+
def dump!
|
30
|
+
offset = 0
|
31
|
+
while offset < @buffer.size
|
32
|
+
uncompressed = @buffer[offset, @block_size]
|
33
|
+
compressed = Snappy.deflate(uncompressed)
|
34
|
+
|
35
|
+
# Uncompressed size (32 bit integer, BE), compressed size (32 bit integer, BE), data.
|
36
|
+
@io << [uncompressed.size, compressed.size, compressed].pack("NNa#{compressed.size}")
|
37
|
+
offset += uncompressed.size
|
38
|
+
end
|
39
|
+
|
40
|
+
@io.flush
|
41
|
+
@buffer.clear
|
42
|
+
end
|
43
|
+
|
44
|
+
alias_method :flush, :dump!
|
45
|
+
|
46
|
+
def close
|
47
|
+
@io.close
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'stringio'
|
5
|
+
require 'snappy/hadoop/reader'
|
6
|
+
require 'snappy/hadoop/writer'
|
7
|
+
|
8
|
+
module Snappy
|
9
|
+
module Hadoop
|
10
|
+
def self.deflate(source, block_size = Snappy::Hadoop::Writer::DEFAULT_BLOCK_SIZE)
|
11
|
+
compressed_io = StringIO.new
|
12
|
+
writer = Snappy::Hadoop::Writer.new(compressed_io)
|
13
|
+
writer << source
|
14
|
+
writer.flush
|
15
|
+
compressed_io.string
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.inflate(source)
|
19
|
+
Snappy::Hadoop::Reader.new(StringIO.new(source)).read
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/snappy/reader.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "snappy/shim"
|
4
|
+
require "snappy/writer"
|
5
5
|
|
6
6
|
module Snappy
|
7
7
|
class Reader
|
@@ -14,32 +14,36 @@ module Snappy
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def each
|
17
|
+
return to_enum unless block_given?
|
18
|
+
|
17
19
|
until @io.eof?
|
18
20
|
if @chunked
|
19
|
-
size = @io.read(4).
|
20
|
-
yield Snappy.inflate(@io.read(size))
|
21
|
+
size = @io.read(4).unpack1("N")
|
22
|
+
yield Snappy.inflate(@io.read(size))
|
21
23
|
else
|
22
|
-
yield Snappy.inflate
|
24
|
+
yield Snappy.inflate(@io.read)
|
23
25
|
end
|
24
26
|
end
|
25
27
|
end
|
26
28
|
|
27
29
|
def read
|
28
|
-
|
30
|
+
buff = StringIO.new
|
29
31
|
each do |chunk|
|
30
|
-
|
32
|
+
buff << chunk
|
31
33
|
end
|
32
|
-
|
34
|
+
buff.string
|
33
35
|
end
|
34
36
|
|
35
37
|
def each_line(sep_string = $/)
|
38
|
+
return to_enum(:each_line, sep_string) unless block_given?
|
39
|
+
|
36
40
|
last = ""
|
37
41
|
each do |chunk|
|
38
42
|
chunk = last + chunk
|
39
43
|
lines = chunk.split(sep_string)
|
40
44
|
last = lines.pop
|
41
45
|
lines.each do |line|
|
42
|
-
yield line
|
46
|
+
yield line
|
43
47
|
end
|
44
48
|
end
|
45
49
|
yield last
|
@@ -50,7 +54,7 @@ module Snappy
|
|
50
54
|
def read_header!
|
51
55
|
header = @io.read Snappy::Writer::MAGIC.length
|
52
56
|
if header.length == Snappy::Writer::MAGIC.length && header == Snappy::Writer::MAGIC
|
53
|
-
@magic, @default_version, @minimum_compatible_version = header, *@io.read(8).unpack(
|
57
|
+
@magic, @default_version, @minimum_compatible_version = header, *@io.read(8).unpack("NN")
|
54
58
|
@chunked = true
|
55
59
|
else
|
56
60
|
@io.rewind
|
data/lib/snappy/shim.rb
CHANGED
data/lib/snappy/version.rb
CHANGED
data/lib/snappy.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
4
|
+
require "snappy-jars" if defined?(JRUBY_VERSION)
|
5
|
+
require "snappy_ext"
|
6
|
+
require "snappy/reader"
|
7
|
+
require "snappy/writer"
|
8
|
+
require "snappy/hadoop"
|
data/snappy.gemspec
CHANGED
@@ -1,36 +1,36 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
5
|
+
require "snappy/version"
|
5
6
|
|
6
7
|
Gem::Specification.new do |spec|
|
7
8
|
spec.name = "snappy"
|
8
9
|
spec.version = Snappy::VERSION
|
9
10
|
spec.authors = ["miyucy"]
|
10
11
|
spec.email = ["fistfvck@gmail.com"]
|
11
|
-
spec.description =
|
12
|
-
spec.summary =
|
12
|
+
spec.description = "libsnappy binding for Ruby"
|
13
|
+
spec.summary = "libsnappy binding for Ruby"
|
13
14
|
spec.homepage = "http://github.com/miyucy/snappy"
|
14
15
|
spec.license = "MIT"
|
15
16
|
|
17
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
18
|
+
spec.metadata["source_code_uri"] = spec.homepage
|
19
|
+
|
16
20
|
spec.test_files = `git ls-files -z -- test`.split("\x0")
|
17
21
|
spec.files = `git ls-files -z`.split("\x0")
|
18
22
|
spec.files -= spec.test_files
|
19
|
-
spec.files -= [
|
20
|
-
spec.files += Dir[
|
23
|
+
spec.files -= ["vendor/snappy"]
|
24
|
+
spec.files += Dir["vendor/snappy/**/*"].reject { |e| e.start_with? "vendor/snappy/testdata" }
|
21
25
|
|
22
26
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
23
27
|
spec.require_paths = ["lib"]
|
24
28
|
|
25
29
|
if defined?(JRUBY_VERSION)
|
26
30
|
spec.files += %w[lib/snappy_ext.jar]
|
27
|
-
spec.platform =
|
28
|
-
spec.add_dependency
|
31
|
+
spec.platform = "java"
|
32
|
+
spec.add_dependency "snappy-jars", "~> 1.1.0"
|
29
33
|
else
|
30
34
|
spec.extensions = ["ext/extconf.rb"]
|
31
35
|
end
|
32
|
-
|
33
|
-
spec.add_development_dependency "bundler", "~> 1.3"
|
34
|
-
spec.add_development_dependency "rake"
|
35
|
-
spec.add_development_dependency "minitest"
|
36
36
|
end
|