snappy 0.0.17 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +5 -5
  2. data/.travis.yml +6 -1
  3. data/Gemfile +5 -0
  4. data/README.md +28 -4
  5. data/Rakefile +1 -0
  6. data/ext/extconf.rb +21 -16
  7. data/lib/snappy.rb +1 -0
  8. data/lib/snappy/hadoop.rb +22 -0
  9. data/lib/snappy/hadoop/reader.rb +58 -0
  10. data/lib/snappy/hadoop/writer.rb +51 -0
  11. data/lib/snappy/reader.rb +4 -4
  12. data/lib/snappy/version.rb +1 -1
  13. data/smoke.sh +1 -1
  14. data/snappy.gemspec +0 -4
  15. data/test/hadoop/test-snappy-hadoop-reader.rb +103 -0
  16. data/test/hadoop/test-snappy-hadoop-writer.rb +48 -0
  17. data/test/test-snappy-hadoop.rb +22 -0
  18. data/vendor/snappy/CMakeLists.txt +174 -0
  19. data/vendor/snappy/CONTRIBUTING.md +26 -0
  20. data/vendor/snappy/NEWS +32 -0
  21. data/vendor/snappy/{README → README.md} +13 -3
  22. data/vendor/snappy/cmake/SnappyConfig.cmake +1 -0
  23. data/vendor/snappy/cmake/config.h.in +62 -0
  24. data/vendor/snappy/snappy-internal.h +9 -12
  25. data/vendor/snappy/snappy-stubs-internal.h +63 -30
  26. data/vendor/snappy/snappy-stubs-public.h.in +13 -19
  27. data/vendor/snappy/snappy-test.cc +10 -7
  28. data/vendor/snappy/snappy-test.h +12 -38
  29. data/vendor/snappy/snappy.cc +81 -30
  30. data/vendor/snappy/snappy_unittest.cc +16 -137
  31. metadata +18 -82
  32. data/vendor/snappy/ChangeLog +0 -2468
  33. data/vendor/snappy/INSTALL +0 -370
  34. data/vendor/snappy/Makefile +0 -982
  35. data/vendor/snappy/Makefile.am +0 -26
  36. data/vendor/snappy/Makefile.in +0 -982
  37. data/vendor/snappy/aclocal.m4 +0 -9738
  38. data/vendor/snappy/autogen.sh +0 -12
  39. data/vendor/snappy/autom4te.cache/output.0 +0 -18856
  40. data/vendor/snappy/autom4te.cache/output.1 +0 -18852
  41. data/vendor/snappy/autom4te.cache/requests +0 -297
  42. data/vendor/snappy/autom4te.cache/traces.0 +0 -2689
  43. data/vendor/snappy/autom4te.cache/traces.1 +0 -714
  44. data/vendor/snappy/config.guess +0 -1530
  45. data/vendor/snappy/config.h +0 -135
  46. data/vendor/snappy/config.h.in +0 -134
  47. data/vendor/snappy/config.log +0 -1640
  48. data/vendor/snappy/config.status +0 -2318
  49. data/vendor/snappy/config.sub +0 -1773
  50. data/vendor/snappy/configure +0 -18852
  51. data/vendor/snappy/configure.ac +0 -134
  52. data/vendor/snappy/depcomp +0 -688
  53. data/vendor/snappy/install-sh +0 -527
  54. data/vendor/snappy/libtool +0 -10246
  55. data/vendor/snappy/ltmain.sh +0 -9661
  56. data/vendor/snappy/m4/gtest.m4 +0 -74
  57. data/vendor/snappy/m4/libtool.m4 +0 -8001
  58. data/vendor/snappy/m4/ltoptions.m4 +0 -384
  59. data/vendor/snappy/m4/ltsugar.m4 +0 -123
  60. data/vendor/snappy/m4/ltversion.m4 +0 -23
  61. data/vendor/snappy/m4/lt~obsolete.m4 +0 -98
  62. data/vendor/snappy/missing +0 -331
  63. data/vendor/snappy/snappy-stubs-public.h +0 -100
  64. data/vendor/snappy/snappy.pc +0 -10
  65. data/vendor/snappy/snappy.pc.in +0 -10
  66. data/vendor/snappy/stamp-h1 +0 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 69133bb0d8adece5bc23fcf0db6e66180c58b04c
4
- data.tar.gz: 410ae9d85c9260000a0b816efe4954a92f4188d9
2
+ SHA256:
3
+ metadata.gz: 3b633f54dc239a8a5a4ef4df91fd0a23a100298ca83192d30096490f6174670b
4
+ data.tar.gz: 7caed929e615860f65dc0a25568cbe1c4f01ed377fbbc96de457acffb9dcc349
5
5
  SHA512:
6
- metadata.gz: 18927ee51e2ddca5bc23c1fa80d207f8b9c2ae9450085606153233dd000c1c1f72bf14f43a2248d1290328bfde776760e68800d10526447b96cd975824b7528c
7
- data.tar.gz: 8f9ed859c2c472a408425d1981c63a92c0c7349d1e1a4426b422dd087d89f70167ed0c80e009f581c0f348d65afa5ef017cb2ee96f3620049f921f0260c09f36
6
+ metadata.gz: 6c51f97d066a0bc256498157c9e9a2dab2da1191f310da24d0b54531de6ceb4db09471a4d8738180e20ca06f8e7147440ded43c6ad6baf1599f626c751a9086a
7
+ data.tar.gz: 8d8b5494df7e5858ebe77c2a86845257409cd71d793f8cd7c2123f9f784e75fb0d66294ce87ddd5fc5140b4702a135e8a644b5a07308414a9fc747fb6111658a
@@ -9,12 +9,17 @@ rvm:
9
9
  - 2.2
10
10
  - 2.3
11
11
  - 2.4
12
+ - 2.5
12
13
  - jruby-19mode
13
14
  env:
14
15
  global:
15
16
  - PATH="/usr/lib/ccache:$PATH"
16
17
  before_install:
17
- - gem install bundler
18
+ - travis_retry gem install bundler --no-document -v 1.16.1
19
+ addons:
20
+ apt:
21
+ packages:
22
+ - cmake
18
23
  deploy:
19
24
  provider: rubygems
20
25
  api_key:
data/Gemfile CHANGED
@@ -2,3 +2,8 @@ source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in snappy.gemspec
4
4
  gemspec
5
+
6
+ group :development do
7
+ gem 'rake'
8
+ gem 'minitest'
9
+ end
data/README.md CHANGED
@@ -4,13 +4,27 @@ see https://github.com/google/snappy
4
4
 
5
5
  ## Preparation
6
6
 
7
- Use libsnappy
8
7
 
9
- $ brew install snappy
8
+ ### macOS
10
9
 
11
- Or
10
+ ```bash
11
+ $ brew install snappy
12
+ $ brew install autoconf automake libtool
13
+ ```
14
+
15
+ ### Ubuntu
16
+
17
+ ```bash
18
+ $ apt-get install libsnappy-dev -y
19
+ $ apt-get install libtool automake autoconf -y
20
+ ```
21
+
22
+ ### Alpine
12
23
 
13
- $ brew install autoconf automake libtool
24
+ ```bash
25
+ $ apk install snappy
26
+ $ apk install build-base libexecinfo automake autoconf libtool
27
+ ```
14
28
 
15
29
  ## Installation
16
30
 
@@ -28,6 +42,7 @@ Or install it yourself as:
28
42
 
29
43
  ## Usage
30
44
 
45
+ Snappy-java format
31
46
  ```ruby
32
47
  require 'snappy'
33
48
 
@@ -38,6 +53,15 @@ Snappy.inflate(source)
38
53
  # => Decompressed data
39
54
  ```
40
55
 
56
+ Hadoop-snappy format
57
+ ```ruby
58
+ Snappy::Hadoop.deflate(source)
59
+ # => Compressed data
60
+
61
+ Snappy::Hadoop.inflate(source)
62
+ # => Decompressed data
63
+ ```
64
+
41
65
  ## Contributing
42
66
 
43
67
  1. Fork it
data/Rakefile CHANGED
@@ -5,6 +5,7 @@ require "rbconfig"
5
5
  DLEXT = RbConfig::CONFIG['DLEXT']
6
6
 
7
7
  Rake::TestTask.new do |t|
8
+ t.pattern = 'test/**/test-*.rb'
8
9
  t.warning = true
9
10
  t.verbose = true
10
11
  end
@@ -6,24 +6,29 @@ unless pkg_config('libsnappy') || have_library('snappy')
6
6
  pwd = File.dirname File.expand_path __FILE__
7
7
  dir = File.join pwd, '..', 'vendor', 'snappy'
8
8
 
9
- Dir.chdir dir do
10
- raise '`autogen.sh` failed' unless system './autogen.sh'
11
- raise '`configure` failed' unless system './configure --disable-option-checking --disable-dependency-tracking --disable-gtest --without-gflags'
9
+ Dir.chdir(dir) do
10
+ FileUtils.mkdir_p 'build'
11
+ Dir.chdir(File.join(dir, 'build')) do
12
+ `cmake .. -DCMAKE_BUILD_TYPE=Release`
13
+ end
12
14
  end
13
15
 
14
- src = %w(
15
- config.h
16
- snappy-c.cc
17
- snappy-c.h
18
- snappy-internal.h
19
- snappy-sinksource.cc
20
- snappy-sinksource.h
21
- snappy-stubs-internal.cc
22
- snappy-stubs-internal.h
23
- snappy-stubs-public.h
24
- snappy.cc
25
- snappy.h
26
- ).map { |e| File.join dir, e }
16
+ src = %w[
17
+ config.h
18
+ snappy-stubs-public.h
19
+ ].map { |e| File.join dir, 'build', e }
20
+ FileUtils.cp src, pwd, :verbose => true
21
+ src = %w[
22
+ snappy-c.cc
23
+ snappy-c.h
24
+ snappy-internal.h
25
+ snappy-sinksource.cc
26
+ snappy-sinksource.h
27
+ snappy.cc
28
+ snappy.h
29
+ snappy-stubs-internal.cc
30
+ snappy-stubs-internal.h
31
+ ].map { |e| File.join dir, e }
27
32
  FileUtils.cp src, pwd, :verbose => true
28
33
  end
29
34
 
@@ -5,3 +5,4 @@ require 'snappy-jars' if defined?(JRUBY_VERSION)
5
5
  require 'snappy_ext'
6
6
  require 'snappy/reader'
7
7
  require 'snappy/writer'
8
+ require 'snappy/hadoop'
@@ -0,0 +1,22 @@
1
+ # encoding: utf-8
2
+ # frozen_string_literal: true
3
+
4
+ require 'stringio'
5
+ require 'snappy/hadoop/reader'
6
+ require 'snappy/hadoop/writer'
7
+
8
+ module Snappy
9
+ module Hadoop
10
+ def self.deflate(source, block_size = Snappy::Hadoop::Writer::DEFAULT_BLOCK_SIZE)
11
+ compressed_io = StringIO.new
12
+ writer = Snappy::Hadoop::Writer.new(compressed_io)
13
+ writer << source
14
+ writer.flush
15
+ compressed_io.string
16
+ end
17
+
18
+ def self.inflate(source)
19
+ Snappy::Hadoop::Reader.new(StringIO.new(source)).read
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'stringio'
4
+ require 'snappy/shim'
5
+
6
+ module Snappy
7
+ module Hadoop
8
+ class Reader
9
+ attr_reader :io
10
+
11
+ def initialize(io)
12
+ @io = Snappy.set_encoding io
13
+ yield self if block_given?
14
+ end
15
+
16
+ def each
17
+ until @io.eof?
18
+ # Uncompressed size (32 bit integer, BE).
19
+ uncompressed_size = @io.read(4).unpack('N').first
20
+
21
+ uncompressed_block_io = StringIO.new
22
+ while uncompressed_block_io.size < uncompressed_size
23
+ # Compressed subblock size (32 bit integer, BE).
24
+ compressed_size = @io.read(4).unpack('N').first
25
+ uncompressed_block_io << Snappy.inflate(@io.read(compressed_size))
26
+ end
27
+
28
+ if uncompressed_block_io.size > uncompressed_size
29
+ raise RuntimeError("Invalid data: expected #{uncompressed_size} bytes, got #{Uncompressed.size}")
30
+ end
31
+
32
+ yield uncompressed_block_io.string if block_given?
33
+ end
34
+ end
35
+
36
+ def read
37
+ buff = StringIO.new
38
+ each do |chunk|
39
+ buff << chunk
40
+ end
41
+ buff.string
42
+ end
43
+
44
+ def each_line(sep_string = $/)
45
+ last = ""
46
+ each do |chunk|
47
+ chunk = last + chunk
48
+ lines = chunk.split(sep_string)
49
+ last = lines.pop
50
+ lines.each do |line|
51
+ yield line if block_given?
52
+ end
53
+ end
54
+ yield last
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'snappy/shim'
4
+
5
+ module Snappy
6
+ module Hadoop
7
+ class Writer
8
+ DEFAULT_BLOCK_SIZE = 256 * 1024
9
+
10
+ attr_reader :io, :block_size
11
+
12
+ def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
13
+ @block_size = block_size
14
+ @buffer = String.new
15
+ @io = Snappy.set_encoding io
16
+ if block_given?
17
+ yield self
18
+ dump!
19
+ end
20
+ end
21
+
22
+ def <<(msg)
23
+ @buffer << msg.to_s
24
+ dump! if @buffer.size >= @block_size
25
+ end
26
+
27
+ alias_method :write, :<<
28
+
29
+ def dump!
30
+ offset = 0
31
+ while offset < @buffer.size
32
+ uncompressed = @buffer[offset, @block_size]
33
+ compressed = Snappy.deflate(uncompressed)
34
+
35
+ # Uncompressed size (32 bit integer, BE), compressed size (32 bit integer, BE), data.
36
+ @io << [uncompressed.size, compressed.size, compressed].pack("NNa#{compressed.size}")
37
+ offset += uncompressed.size
38
+ end
39
+
40
+ @io.flush
41
+ @buffer.clear
42
+ end
43
+
44
+ alias_method :flush, :dump!
45
+
46
+ def close
47
+ @io.close
48
+ end
49
+ end
50
+ end
51
+ end
@@ -19,17 +19,17 @@ module Snappy
19
19
  size = @io.read(4).unpack('N').first
20
20
  yield Snappy.inflate(@io.read(size)) if block_given?
21
21
  else
22
- yield Snappy.inflate @io.read if block_given?
22
+ yield Snappy.inflate(@io.read) if block_given?
23
23
  end
24
24
  end
25
25
  end
26
26
 
27
27
  def read
28
- @buff = StringIO.new
28
+ buff = StringIO.new
29
29
  each do |chunk|
30
- @buff << chunk
30
+ buff << chunk
31
31
  end
32
- @buff.string
32
+ buff.string
33
33
  end
34
34
 
35
35
  def each_line(sep_string = $/)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Snappy
4
- VERSION = "0.0.17"
4
+ VERSION = "0.1.0"
5
5
  end
data/smoke.sh CHANGED
@@ -1,7 +1,7 @@
1
1
  #!/bin/bash
2
2
  gem list | grep snappy && gem uninstall --force snappy
3
3
  bundle exec rake clean build
4
- gem install --no-ri --no-rdoc "$(ls pkg/snappy-*.gem)"
4
+ gem install --no-document "$(ls pkg/snappy-*.gem)"
5
5
  cat <<EOF | ruby
6
6
  require 'snappy'
7
7
  abort if Snappy.inflate(Snappy.deflate(File.read('smoke.sh'))) != File.read('smoke.sh')
@@ -29,8 +29,4 @@ Gem::Specification.new do |spec|
29
29
  else
30
30
  spec.extensions = ["ext/extconf.rb"]
31
31
  end
32
-
33
- spec.add_development_dependency "bundler", "~> 1.3"
34
- spec.add_development_dependency "rake"
35
- spec.add_development_dependency "minitest"
36
32
  end
@@ -0,0 +1,103 @@
1
+ require "minitest/autorun"
2
+ require "minitest/spec"
3
+ require "snappy"
4
+ require "stringio"
5
+
6
+ describe Snappy::Hadoop::Reader do
7
+ before do
8
+ @buffer = StringIO.new
9
+ Snappy::Hadoop::Writer.new @buffer do |w|
10
+ w << "foo"
11
+ w << "bar"
12
+ w << "baz"
13
+ w << "quux"
14
+ end
15
+ @buffer.rewind
16
+ end
17
+
18
+ subject do
19
+ Snappy::Hadoop::Reader.new @buffer
20
+ end
21
+
22
+ describe :initialize do
23
+ it "should yield itself to the block" do
24
+ yielded = nil
25
+ returned = Snappy::Hadoop::Reader.new @buffer do |r|
26
+ yielded = r
27
+ end
28
+ returned.must_equal yielded
29
+ end
30
+ end
31
+
32
+ describe :io do
33
+ it "should be a constructor argument" do
34
+ subject.io.must_equal @buffer
35
+ end
36
+
37
+ it "should not receive `length' in initializing" do
38
+ length = MiniTest::Mock.new.expect(:call, 0)
39
+ @buffer.stub(:length, length) do
40
+ Snappy::Hadoop::Reader.new @buffer
41
+ end
42
+ -> { length.verify }.must_raise MockExpectationError
43
+ end
44
+ end
45
+
46
+ describe :each do
47
+ before do
48
+ Snappy::Hadoop::Writer.new @buffer do |w|
49
+ w << "foo"
50
+ w << "bar"
51
+ w.dump!
52
+ w << "baz"
53
+ w << "quux"
54
+ end
55
+ @buffer.rewind
56
+ end
57
+
58
+ it "should yield each chunk" do
59
+ chunks = []
60
+ Snappy::Hadoop::Reader.new(@buffer).each do |chunk|
61
+ chunks << chunk
62
+ end
63
+ chunks.must_equal ["foobar", "bazquux"]
64
+ end
65
+ end
66
+
67
+ describe :read do
68
+ before do
69
+ Snappy::Hadoop::Writer.new @buffer do |w|
70
+ w << "foo"
71
+ w << "bar"
72
+ w << "baz"
73
+ w << "quux"
74
+ end
75
+ @buffer.rewind
76
+ end
77
+
78
+ it "should return the bytes" do
79
+ Snappy::Hadoop::Reader.new(@buffer).read.must_equal "foobarbazquux"
80
+ end
81
+ end
82
+
83
+ describe :each_line do
84
+ before do
85
+ Snappy::Hadoop::Writer.new @buffer do |w|
86
+ w << "foo\n"
87
+ w << "bar"
88
+ w.dump!
89
+ w << "baz\n"
90
+ w << "quux\n"
91
+ end
92
+ @buffer.rewind
93
+ end
94
+
95
+ it "should yield each line" do
96
+ lines = []
97
+ Snappy::Hadoop::Reader.new(@buffer).each_line do |line|
98
+ lines << line
99
+ end
100
+ lines.must_equal ["foo", "barbaz", "quux"]
101
+ end
102
+ end
103
+ end