snappy 0.1.0-java → 0.2.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/main.yml +34 -0
  3. data/.github/workflows/publish.yml +34 -0
  4. data/Gemfile +3 -4
  5. data/Rakefile +32 -30
  6. data/ext/api.c +6 -1
  7. data/lib/snappy.rb +5 -5
  8. data/lib/snappy/hadoop/reader.rb +6 -2
  9. data/lib/snappy/reader.rb +11 -7
  10. data/lib/snappy/shim.rb +1 -1
  11. data/lib/snappy/version.rb +1 -1
  12. data/snappy.gemspec +13 -9
  13. data/test/hadoop/snappy_hadoop_reader_test.rb +115 -0
  14. data/test/hadoop/snappy_hadoop_writer_test.rb +48 -0
  15. data/test/snappy_hadoop_test.rb +26 -0
  16. data/test/snappy_reader_test.rb +148 -0
  17. data/test/snappy_test.rb +95 -0
  18. data/test/snappy_writer_test.rb +55 -0
  19. data/test/test_helper.rb +7 -0
  20. data/vendor/snappy/CMakeLists.txt +177 -54
  21. data/vendor/snappy/NEWS +8 -0
  22. data/vendor/snappy/README.md +19 -20
  23. data/vendor/snappy/cmake/SnappyConfig.cmake.in +33 -0
  24. data/vendor/snappy/cmake/config.h.in +6 -6
  25. data/vendor/snappy/docs/README.md +72 -0
  26. data/vendor/snappy/snappy-internal.h +12 -5
  27. data/vendor/snappy/snappy-stubs-internal.cc +1 -1
  28. data/vendor/snappy/snappy-stubs-internal.h +60 -15
  29. data/vendor/snappy/snappy-stubs-public.h.in +16 -36
  30. data/vendor/snappy/snappy-test.cc +16 -15
  31. data/vendor/snappy/snappy-test.h +12 -60
  32. data/vendor/snappy/snappy.cc +333 -187
  33. data/vendor/snappy/snappy.h +14 -10
  34. data/vendor/snappy/snappy_compress_fuzzer.cc +59 -0
  35. data/vendor/snappy/snappy_uncompress_fuzzer.cc +57 -0
  36. data/vendor/snappy/snappy_unittest.cc +220 -124
  37. metadata +25 -18
  38. data/.travis.yml +0 -31
  39. data/smoke.sh +0 -8
  40. data/test/hadoop/test-snappy-hadoop-reader.rb +0 -103
  41. data/test/hadoop/test-snappy-hadoop-writer.rb +0 -48
  42. data/test/test-snappy-hadoop.rb +0 -22
  43. data/test/test-snappy-reader.rb +0 -129
  44. data/test/test-snappy-writer.rb +0 -55
  45. data/test/test-snappy.rb +0 -58
  46. data/vendor/snappy/cmake/SnappyConfig.cmake +0 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 40598da197be3dd37207207288f7d07e864db1854075e05083aab32d0f36c2e7
4
- data.tar.gz: 4e530705d88db960c3b0bd3474d33670ae0e79d55d0202770557fca817798efb
3
+ metadata.gz: dc1e9d110a0bf1b0a6e04906ed2ed31223b95ed7b009c684b8e499a78b1a17e8
4
+ data.tar.gz: 325f0fa94023b89a4ff06d6bb4cde178fef16f9d692e6f331227af3ee31b41b8
5
5
  SHA512:
6
- metadata.gz: 27e1b4cc2d86dff4357130204a94c6212edd8999379be5dcd282ecdad59359ecefded8bc6bb122160683e2a005f8f1d8c229cc2e151c29eaaca0f6967bdd4901
7
- data.tar.gz: 240cab9f029fdbe3132bfa76f846135b1d38d9661af81e353650a0674d62333a1025b8bbbcb291167ceca704d6ae10487cb78eb47009aac689af93542f5be474
6
+ metadata.gz: 01bcc433736c9f9bf08d0cad2eb5647bc771099490c3a60f67a629a16acd8885179718f16f59f126610658292ba86a1a9058eb1243d81077efebe6d523895a57
7
+ data.tar.gz: 9eaf7fff9956c56c6db98ba4b38370d5ef27c1ad1b97bb4437006092b7c78bd5d7c52f23b42bf1bcfc7f69a06a82fb288d1e42b277805d8d172850cf520537b4
@@ -0,0 +1,34 @@
1
+ name: Ruby
2
+
3
+ on: [pull_request]
4
+
5
+ jobs:
6
+ build:
7
+ strategy:
8
+ fail-fast: false
9
+ matrix:
10
+ ruby: [2.5, 2.6, 2.7, jruby]
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - uses: actions/checkout@v2
14
+ with:
15
+ submodules: true
16
+ - name: Set up Ruby
17
+ uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ bundler-cache: true
21
+ - name: Run the default task
22
+ run: |
23
+ bundle exec rake clobber clean test build
24
+ gem install --no-document "$(ls pkg/snappy-*.gem)"
25
+ cat <<EOF | ruby
26
+ require "snappy"
27
+ if Snappy.inflate(Snappy.deflate(File.read("README.md"))) == File.read("README.md")
28
+ puts "OK"
29
+ exit 0
30
+ else
31
+ puts "NG"
32
+ exit 0
33
+ end
34
+ EOF
@@ -0,0 +1,34 @@
1
+ name: Publish Gem
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - 'v*'
7
+
8
+ jobs:
9
+ build:
10
+ strategy:
11
+ fail-fast: false
12
+ matrix:
13
+ ruby: [2.7, jruby]
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ with:
18
+ submodules: true
19
+ - name: Set up Ruby
20
+ uses: ruby/setup-ruby@v1
21
+ with:
22
+ ruby-version: ${{ matrix.ruby }}
23
+ bundler-cache: true
24
+ - name: Run release task
25
+ run: |
26
+ mkdir -p ~/.gem
27
+ cat << EOF > ~/.gem/credentials
28
+ ---
29
+ :github: Bearer ${{secrets.GITHUB_TOKEN}}
30
+ :rubygems_api_key: ${{secrets.RUBYGEMS_API_KEY}}
31
+ EOF
32
+ chmod 600 ~/.gem/credentials
33
+ bundle exec rake release[remote]
34
+ rm -f ~/.gem/credentials
data/Gemfile CHANGED
@@ -3,7 +3,6 @@ source 'https://rubygems.org'
3
3
  # Specify your gem's dependencies in snappy.gemspec
4
4
  gemspec
5
5
 
6
- group :development do
7
- gem 'rake'
8
- gem 'minitest'
9
- end
6
+ gem "rake", "~> 13.0"
7
+ gem "test-unit", "~> 3.0"
8
+ gem "test-unit-rr"
data/Rakefile CHANGED
@@ -2,49 +2,51 @@ require "bundler/setup"
2
2
  require "bundler/gem_tasks"
3
3
  require "rake/testtask"
4
4
  require "rbconfig"
5
- DLEXT = RbConfig::CONFIG['DLEXT']
5
+ DLEXT = RbConfig::CONFIG["DLEXT"]
6
6
 
7
- Rake::TestTask.new do |t|
8
- t.pattern = 'test/**/test-*.rb'
7
+ Rake::TestTask.new(:test) do |t|
8
+ t.libs << "test"
9
+ t.libs << "lib"
10
+ t.test_files = FileList["test/**/*_test.rb"]
9
11
  t.warning = true
10
12
  t.verbose = true
11
13
  end
12
14
 
13
15
  if defined?(JRUBY_VERSION)
14
- require 'ant'
16
+ require "ant"
15
17
 
16
- directory 'ext/java/build'
18
+ directory "ext/java/build"
17
19
 
18
- task :setup => 'ext/java/build' do
19
- ant.property name: 'src.dir', value: 'ext/java/src'
20
- ant.property name: 'build.dir', value: 'ext/java/build'
20
+ task setup: "ext/java/build" do
21
+ ant.property name: "src.dir", value: "ext/java/src"
22
+ ant.property name: "build.dir", value: "ext/java/build"
21
23
 
22
- ant.path id: 'compile.class.path' do
23
- pathelement location: File.join(RbConfig::CONFIG['prefix'], 'lib', 'jruby.jar')
24
- $LOAD_PATH.flat_map { |path| Dir[File.join(path, '**', '*.jar')] }.each do |jar|
24
+ ant.path id: "compile.class.path" do
25
+ pathelement location: File.join(RbConfig::CONFIG["prefix"], "lib", "jruby.jar")
26
+ $LOAD_PATH.flat_map { |path| Dir[File.join(path, "**", "*.jar")] }.each do |jar|
25
27
  pathelement location: jar
26
28
  end
27
29
  end
28
30
  end
29
31
 
30
- desc 'Compile the extension'
31
- task :compile => :setup do
32
- ant.javac destdir: '${build.dir}', includeantruntime: 'no', target: '1.6', source: '1.6', debug: 'on' do
33
- classpath refid: 'compile.class.path'
34
- src { pathelement location: '${src.dir}' }
32
+ desc "Compile the extension"
33
+ task compile: :setup do
34
+ ant.javac destdir: "${build.dir}", includeantruntime: "no", target: "1.6", source: "1.6", debug: "on" do
35
+ classpath refid: "compile.class.path"
36
+ src { pathelement location: "${src.dir}" }
35
37
  end
36
38
  end
37
39
 
38
- desc 'Package the jar'
39
- file 'lib/snappy_ext.jar' => :compile do |t|
40
- ant.jar destfile: 'lib/snappy_ext.jar', basedir: '${build.dir}' do
41
- ant.fileset dir: '${build.dir}', includes: 'snappy/*.class'
42
- ant.fileset dir: '${build.dir}', includes: 'SnappyExtLibraryService.class'
40
+ desc "Package the jar"
41
+ file "lib/snappy_ext.jar" => :compile do |t|
42
+ ant.jar destfile: "lib/snappy_ext.jar", basedir: "${build.dir}" do
43
+ ant.fileset dir: "${build.dir}", includes: "snappy/*.class"
44
+ ant.fileset dir: "${build.dir}", includes: "SnappyExtLibraryService.class"
43
45
  end
44
46
  end
45
47
 
46
- task :test => 'lib/snappy_ext.jar'
47
- task :build => [:clean, 'lib/snappy_ext.jar']
48
+ task test: "lib/snappy_ext.jar"
49
+ task build: [:clean, "lib/snappy_ext.jar"]
48
50
  else
49
51
  file "ext/snappy_ext.#{DLEXT}" => Dir.glob("ext/*{.rb,.c}") do
50
52
  Dir.chdir("ext") do
@@ -54,15 +56,15 @@ else
54
56
  cp "ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}"
55
57
  end
56
58
 
57
-
58
- task :test => "ext/snappy_ext.#{DLEXT}"
59
+ task test: "ext/snappy_ext.#{DLEXT}"
59
60
  end
60
61
 
61
- desc 'Clean up build artifacts'
62
+ desc "Clean up build artifacts"
62
63
  task :clean do
63
- rm_rf 'ext/java/build'
64
- rm_rf 'lib/snappy_ext.jar'
65
- rm_rf(["ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}", 'ext/mkmf.log', 'ext/config.h', 'ext/api.o', 'ext/Makefile', 'ext/snappy.cc', 'ext/snappy.h', 'ext/snappy.o'] + Dir['ext/snappy-*'])
64
+ rm_rf "ext/java/build"
65
+ rm_rf "lib/snappy_ext.jar"
66
+ rm_rf(["ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}", "ext/mkmf.log", "ext/config.h", "ext/api.o",
67
+ "ext/Makefile", "ext/snappy.cc", "ext/snappy.h", "ext/snappy.o"] + Dir["ext/snappy-*"])
66
68
  end
67
69
 
68
- task :default => :test
70
+ task default: :test
data/ext/api.c CHANGED
@@ -1,7 +1,6 @@
1
1
  #include "ruby.h"
2
2
  #include "snappy-c.h"
3
3
 
4
- static VALUE rb_mSnappy;
5
4
  static VALUE rb_eSnappy;
6
5
 
7
6
  static VALUE
@@ -92,10 +91,16 @@ snappy_valid_p(VALUE self, VALUE str)
92
91
 
93
92
  void Init_snappy_ext()
94
93
  {
94
+ #if HAVE_RB_EXT_RACTOR_SAFE
95
+ rb_ext_ractor_safe(true);
96
+ #endif
97
+
98
+ VALUE rb_mSnappy;
95
99
  VALUE rb_mSnappy_singleton;
96
100
 
97
101
  rb_mSnappy = rb_define_module("Snappy");
98
102
  rb_eSnappy = rb_define_class_under(rb_mSnappy, "Error", rb_eStandardError);
103
+ rb_global_variable(&rb_eSnappy);
99
104
  rb_define_singleton_method(rb_mSnappy, "deflate", snappy_deflate, -1);
100
105
  rb_define_singleton_method(rb_mSnappy, "inflate", snappy_inflate, -1);
101
106
  rb_define_singleton_method(rb_mSnappy, "valid?", snappy_valid_p, 1);
@@ -1,8 +1,8 @@
1
1
  # encoding: utf-8
2
2
  # frozen_string_literal: true
3
3
 
4
- require 'snappy-jars' if defined?(JRUBY_VERSION)
5
- require 'snappy_ext'
6
- require 'snappy/reader'
7
- require 'snappy/writer'
8
- require 'snappy/hadoop'
4
+ require "snappy-jars" if defined?(JRUBY_VERSION)
5
+ require "snappy_ext"
6
+ require "snappy/reader"
7
+ require "snappy/writer"
8
+ require "snappy/hadoop"
@@ -14,6 +14,8 @@ module Snappy
14
14
  end
15
15
 
16
16
  def each
17
+ return to_enum unless block_given?
18
+
17
19
  until @io.eof?
18
20
  # Uncompressed size (32 bit integer, BE).
19
21
  uncompressed_size = @io.read(4).unpack('N').first
@@ -29,7 +31,7 @@ module Snappy
29
31
  raise RuntimeError("Invalid data: expected #{uncompressed_size} bytes, got #{Uncompressed.size}")
30
32
  end
31
33
 
32
- yield uncompressed_block_io.string if block_given?
34
+ yield uncompressed_block_io.string
33
35
  end
34
36
  end
35
37
 
@@ -42,13 +44,15 @@ module Snappy
42
44
  end
43
45
 
44
46
  def each_line(sep_string = $/)
47
+ return to_enum(:each_line, sep_string) unless block_given?
48
+
45
49
  last = ""
46
50
  each do |chunk|
47
51
  chunk = last + chunk
48
52
  lines = chunk.split(sep_string)
49
53
  last = lines.pop
50
54
  lines.each do |line|
51
- yield line if block_given?
55
+ yield line
52
56
  end
53
57
  end
54
58
  yield last
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'snappy/shim'
4
- require 'snappy/writer'
3
+ require "snappy/shim"
4
+ require "snappy/writer"
5
5
 
6
6
  module Snappy
7
7
  class Reader
@@ -14,12 +14,14 @@ module Snappy
14
14
  end
15
15
 
16
16
  def each
17
+ return to_enum unless block_given?
18
+
17
19
  until @io.eof?
18
20
  if @chunked
19
- size = @io.read(4).unpack('N').first
20
- yield Snappy.inflate(@io.read(size)) if block_given?
21
+ size = @io.read(4).unpack1("N")
22
+ yield Snappy.inflate(@io.read(size))
21
23
  else
22
- yield Snappy.inflate(@io.read) if block_given?
24
+ yield Snappy.inflate(@io.read)
23
25
  end
24
26
  end
25
27
  end
@@ -33,13 +35,15 @@ module Snappy
33
35
  end
34
36
 
35
37
  def each_line(sep_string = $/)
38
+ return to_enum(:each_line, sep_string) unless block_given?
39
+
36
40
  last = ""
37
41
  each do |chunk|
38
42
  chunk = last + chunk
39
43
  lines = chunk.split(sep_string)
40
44
  last = lines.pop
41
45
  lines.each do |line|
42
- yield line if block_given?
46
+ yield line
43
47
  end
44
48
  end
45
49
  yield last
@@ -50,7 +54,7 @@ module Snappy
50
54
  def read_header!
51
55
  header = @io.read Snappy::Writer::MAGIC.length
52
56
  if header.length == Snappy::Writer::MAGIC.length && header == Snappy::Writer::MAGIC
53
- @magic, @default_version, @minimum_compatible_version = header, *@io.read(8).unpack('NN')
57
+ @magic, @default_version, @minimum_compatible_version = header, *@io.read(8).unpack("NN")
54
58
  @chunked = true
55
59
  else
56
60
  @io.rewind
@@ -3,7 +3,7 @@
3
3
  module Snappy
4
4
  module_function
5
5
 
6
- if RUBY_VERSION[0..2] == '1.8'
6
+ if RUBY_VERSION[0..2] == "1.8"
7
7
  def set_encoding(io)
8
8
  io
9
9
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Snappy
4
- VERSION = "0.1.0"
4
+ VERSION = "0.2.0"
5
5
  end
@@ -1,31 +1,35 @@
1
- # coding: utf-8
2
- lib = File.expand_path('../lib', __FILE__)
1
+ # frozen_string_literal: true
2
+
3
+ lib = File.expand_path("lib", __dir__)
3
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
- require 'snappy/version'
5
+ require "snappy/version"
5
6
 
6
7
  Gem::Specification.new do |spec|
7
8
  spec.name = "snappy"
8
9
  spec.version = Snappy::VERSION
9
10
  spec.authors = ["miyucy"]
10
11
  spec.email = ["fistfvck@gmail.com"]
11
- spec.description = %q{libsnappy binding for Ruby}
12
- spec.summary = %q{libsnappy binding for Ruby}
12
+ spec.description = "libsnappy binding for Ruby"
13
+ spec.summary = "libsnappy binding for Ruby"
13
14
  spec.homepage = "http://github.com/miyucy/snappy"
14
15
  spec.license = "MIT"
15
16
 
17
+ spec.metadata["homepage_uri"] = spec.homepage
18
+ spec.metadata["source_code_uri"] = spec.homepage
19
+
16
20
  spec.test_files = `git ls-files -z -- test`.split("\x0")
17
21
  spec.files = `git ls-files -z`.split("\x0")
18
22
  spec.files -= spec.test_files
19
- spec.files -= ['vendor/snappy']
20
- spec.files += Dir['vendor/snappy/**/*'].reject { |e| e.start_with? 'vendor/snappy/testdata' }
23
+ spec.files -= ["vendor/snappy"]
24
+ spec.files += Dir["vendor/snappy/**/*"].reject { |e| e.start_with? "vendor/snappy/testdata" }
21
25
 
22
26
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
23
27
  spec.require_paths = ["lib"]
24
28
 
25
29
  if defined?(JRUBY_VERSION)
26
30
  spec.files += %w[lib/snappy_ext.jar]
27
- spec.platform = 'java'
28
- spec.add_dependency 'snappy-jars', '~> 1.1.0'
31
+ spec.platform = "java"
32
+ spec.add_dependency "snappy-jars", "~> 1.1.0"
29
33
  else
30
34
  spec.extensions = ["ext/extconf.rb"]
31
35
  end
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "test_helper"
4
+ require "stringio"
5
+
6
+ class SnappyHadoopReaderTest < Test::Unit::TestCase
7
+ def setup
8
+ @buffer = StringIO.new
9
+ Snappy::Hadoop::Writer.new @buffer do |w|
10
+ w << "foo"
11
+ w << "bar"
12
+ w << "baz"
13
+ w << "quux"
14
+ end
15
+ @buffer.rewind
16
+ end
17
+
18
+ def subject
19
+ @subject ||= Snappy::Hadoop::Reader.new @buffer
20
+ end
21
+
22
+ sub_test_case "#initialize" do
23
+ test "should yield itself to the block" do
24
+ yielded = nil
25
+ returned = Snappy::Hadoop::Reader.new @buffer do |r|
26
+ yielded = r
27
+ end
28
+ assert_equal returned, yielded
29
+ end
30
+ end
31
+
32
+ sub_test_case "#io" do
33
+ test "should be a constructor argument" do
34
+ assert_equal @buffer, subject.io
35
+ end
36
+
37
+ test "should not receive `length' in initializing" do
38
+ dont_allow(@buffer).length
39
+ Snappy::Hadoop::Reader.new @buffer
40
+ end
41
+ end
42
+
43
+ sub_test_case "#each" do
44
+ def setup
45
+ @buffer = StringIO.new
46
+ Snappy::Hadoop::Writer.new @buffer do |w|
47
+ w << "foo"
48
+ w << "bar"
49
+ w.dump!
50
+ w << "baz"
51
+ w << "quux"
52
+ end
53
+ @buffer.rewind
54
+ end
55
+
56
+ test "should yield each chunk" do
57
+ chunks = []
58
+ subject.each do |chunk|
59
+ chunks << chunk
60
+ end
61
+ assert_equal %w[foobar bazquux], chunks
62
+ end
63
+ end
64
+
65
+ sub_test_case "#read" do
66
+ test "should return the bytes" do
67
+ assert_equal "foobarbazquux", subject.read
68
+ end
69
+ end
70
+
71
+ sub_test_case "#each_line" do
72
+ def setup
73
+ @buffer = StringIO.new
74
+ Snappy::Hadoop::Writer.new @buffer do |w|
75
+ w << "foo\n"
76
+ w << "bar"
77
+ w.dump!
78
+ w << "baz\n"
79
+ w << "quux\n"
80
+ end
81
+ @buffer.rewind
82
+ end
83
+
84
+ test "should yield each line" do
85
+ lines = []
86
+ subject.each_line do |line|
87
+ lines << line
88
+ end
89
+ assert_equal %w[foo barbaz quux], lines
90
+ end
91
+
92
+ test "should return enumerator w/o block" do
93
+ eacher = subject.each_line
94
+ assert_instance_of Enumerator, eacher
95
+ lines = []
96
+ loop { lines << eacher.next }
97
+ assert_equal %w[foo barbaz quux], lines
98
+ end
99
+
100
+ test "each_line split by sep_string" do
101
+ buffer = StringIO.new
102
+ Snappy::Hadoop::Writer.new buffer do |w|
103
+ w << %w[a b c].join(",")
104
+ w.dump!
105
+ w << %w[d e f].join(",")
106
+ end
107
+ buffer.rewind
108
+ reader = Snappy::Hadoop::Reader.new buffer
109
+ eacher = reader.each_line(",")
110
+ lines = []
111
+ loop { lines << eacher.next }
112
+ assert_equal %w[a b cd e f], lines
113
+ end
114
+ end
115
+ end