snappy 0.0.14-java → 0.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e0dd553a2b05494e3985d5b71be1c0381468b011
-  data.tar.gz: 2a146b0103be309d4c9ceb3ae51481fd2e218958
+SHA256:
+  metadata.gz: dc1e9d110a0bf1b0a6e04906ed2ed31223b95ed7b009c684b8e499a78b1a17e8
+  data.tar.gz: 325f0fa94023b89a4ff06d6bb4cde178fef16f9d692e6f331227af3ee31b41b8
 SHA512:
-  metadata.gz: 9a721d16f5f799600374314dd118a465a41e6c5d2ea432095b4bfe7533f37989256b9f9b482d26c036424b847119cb8d203edb839fafc618785da628ac944232
-  data.tar.gz: 1e70c72896879655be67cd11a32511342d439e3ed98121d4fd37238c2f62a84d032180065f859b5d8f0e53b8559322e886e3daa9c3dabd7ee998f5925340367d
+  metadata.gz: 01bcc433736c9f9bf08d0cad2eb5647bc771099490c3a60f67a629a16acd8885179718f16f59f126610658292ba86a1a9058eb1243d81077efebe6d523895a57
+  data.tar.gz: 9eaf7fff9956c56c6db98ba4b38370d5ef27c1ad1b97bb4437006092b7c78bd5d7c52f23b42bf1bcfc7f69a06a82fb288d1e42b277805d8d172850cf520537b4

data/.github/workflows/main.yml

@@ -0,0 +1,34 @@
+name: Ruby
+
+on: [pull_request]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.5, 2.6, 2.7, jruby]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Run the default task
+      run: |
+        bundle exec rake clobber clean test build
+        gem install --no-document "$(ls pkg/snappy-*.gem)"
+        cat <<EOF | ruby
+        require "snappy"
+        if Snappy.inflate(Snappy.deflate(File.read("README.md"))) == File.read("README.md")
+          puts "OK"
+          exit 0
+        else
+          puts "NG"
+          exit 0
+        end
+        EOF

data/.github/workflows/publish.yml

@@ -0,0 +1,34 @@
+name: Publish Gem
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, jruby]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Run release task
+      run: |
+        mkdir -p ~/.gem
+        cat << EOF > ~/.gem/credentials
+        ---
+        :github: Bearer ${{secrets.GITHUB_TOKEN}}
+        :rubygems_api_key: ${{secrets.RUBYGEMS_API_KEY}}
+        EOF
+        chmod 600 ~/.gem/credentials
+        bundle exec rake release[remote]
+        rm -f ~/.gem/credentials

data/Gemfile

@@ -2,3 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in snappy.gemspec
 gemspec
+
+gem "rake", "~> 13.0"
+gem "test-unit", "~> 3.0"
+gem "test-unit-rr"

data/README.md

@@ -4,13 +4,27 @@ see https://github.com/google/snappy
 
 ## Preparation
 
-Use libsnappy
 
-    $ brew install snappy
+### macOS
 
-Or
+```bash
+$ brew install snappy
+$ brew install autoconf automake libtool
+```
+
+### Ubuntu
+
+```bash
+$ apt-get install libsnappy-dev -y
+$ apt-get install libtool automake autoconf -y
+```
+
+### Alpine
 
-    $ brew install autoconf automake libtool
+```bash
+$ apk install snappy
+$ apk install build-base libexecinfo automake autoconf libtool
+```
 
 ## Installation
 
@@ -28,6 +42,7 @@ Or install it yourself as:
 
 ## Usage
 
+Snappy-java format
 ```ruby
 require 'snappy'
 
@@ -38,6 +53,15 @@ Snappy.inflate(source)
 # => Decompressed data
 ```
 
+Hadoop-snappy format
+```ruby
+Snappy::Hadoop.deflate(source)
+# => Compressed data
+
+Snappy::Hadoop.inflate(source)
+# => Decompressed data
+```
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -2,48 +2,51 @@ require "bundler/setup"
 require "bundler/gem_tasks"
 require "rake/testtask"
 require "rbconfig"
-DLEXT = RbConfig::CONFIG['DLEXT']
+DLEXT = RbConfig::CONFIG["DLEXT"]
 
-Rake::TestTask.new do |t|
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
   t.warning = true
   t.verbose = true
 end
 
 if defined?(JRUBY_VERSION)
-  require 'ant'
+  require "ant"
 
-  directory 'ext/java/build'
+  directory "ext/java/build"
 
-  task :setup => 'ext/java/build' do
-    ant.property name: 'src.dir', value: 'ext/java/src'
-    ant.property name: 'build.dir', value: 'ext/java/build'
+  task setup: "ext/java/build" do
+    ant.property name: "src.dir", value: "ext/java/src"
+    ant.property name: "build.dir", value: "ext/java/build"
 
-    ant.path id: 'compile.class.path' do
-      pathelement location: File.join(RbConfig::CONFIG['prefix'], 'lib', 'jruby.jar')
-      $LOAD_PATH.flat_map { |path| Dir[File.join(path, '**', '*.jar')] }.each do |jar|
+    ant.path id: "compile.class.path" do
+      pathelement location: File.join(RbConfig::CONFIG["prefix"], "lib", "jruby.jar")
+      $LOAD_PATH.flat_map { |path| Dir[File.join(path, "**", "*.jar")] }.each do |jar|
         pathelement location: jar
       end
     end
   end
 
-  desc 'Compile the extension'
-  task :compile => :setup do
-    ant.javac destdir: '${build.dir}', includeantruntime: 'no', target: '1.6', source: '1.6', debug: 'on' do
-      classpath refid: 'compile.class.path'
-      src { pathelement location: '${src.dir}' }
+  desc "Compile the extension"
+  task compile: :setup do
+    ant.javac destdir: "${build.dir}", includeantruntime: "no", target: "1.6", source: "1.6", debug: "on" do
+      classpath refid: "compile.class.path"
+      src { pathelement location: "${src.dir}" }
     end
   end
 
-  desc 'Package the jar'
-  file 'lib/snappy_ext.jar' => :compile do |t|
-    ant.jar destfile: 'lib/snappy_ext.jar', basedir: '${build.dir}' do
-      ant.fileset dir: '${build.dir}', includes: 'snappy/*.class'
-      ant.fileset dir: '${build.dir}', includes: 'SnappyExtLibraryService.class'
+  desc "Package the jar"
+  file "lib/snappy_ext.jar" => :compile do |t|
+    ant.jar destfile: "lib/snappy_ext.jar", basedir: "${build.dir}" do
+      ant.fileset dir: "${build.dir}", includes: "snappy/*.class"
+      ant.fileset dir: "${build.dir}", includes: "SnappyExtLibraryService.class"
     end
   end
 
-  task :test => 'lib/snappy_ext.jar'
-  task :build => [:clean, 'lib/snappy_ext.jar']
+  task test: "lib/snappy_ext.jar"
+  task build: [:clean, "lib/snappy_ext.jar"]
 else
   file "ext/snappy_ext.#{DLEXT}" => Dir.glob("ext/*{.rb,.c}") do
     Dir.chdir("ext") do
@@ -53,15 +56,15 @@ else
     cp "ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}"
   end
 
-
-  task :test => "ext/snappy_ext.#{DLEXT}"
+  task test: "ext/snappy_ext.#{DLEXT}"
 end
 
-desc 'Clean up build artifacts'
+desc "Clean up build artifacts"
 task :clean do
-  rm_rf 'ext/java/build'
-  rm_rf 'lib/snappy_ext.jar'
-  rm_rf(["ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}", 'ext/mkmf.log', 'ext/config.h', 'ext/api.o', 'ext/Makefile', 'ext/snappy.cc', 'ext/snappy.h', 'ext/snappy.o'] + Dir['ext/snappy-*'])
+  rm_rf "ext/java/build"
+  rm_rf "lib/snappy_ext.jar"
+  rm_rf(["ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}", "ext/mkmf.log", "ext/config.h", "ext/api.o",
+         "ext/Makefile", "ext/snappy.cc", "ext/snappy.h", "ext/snappy.o"] + Dir["ext/snappy-*"])
 end
 
-task :default => :test
+task default: :test

data/ext/api.c

@@ -1,7 +1,6 @@
 #include "ruby.h"
 #include "snappy-c.h"
 
-static VALUE rb_mSnappy;
 static VALUE rb_eSnappy;
 
 static VALUE
@@ -92,10 +91,16 @@ snappy_valid_p(VALUE self, VALUE str)
 
 void Init_snappy_ext()
 {
+#if HAVE_RB_EXT_RACTOR_SAFE
+    rb_ext_ractor_safe(true);
+#endif
+
+    VALUE rb_mSnappy;
     VALUE rb_mSnappy_singleton;
 
     rb_mSnappy = rb_define_module("Snappy");
     rb_eSnappy = rb_define_class_under(rb_mSnappy, "Error", rb_eStandardError);
+    rb_global_variable(&rb_eSnappy);
     rb_define_singleton_method(rb_mSnappy, "deflate", snappy_deflate, -1);
     rb_define_singleton_method(rb_mSnappy, "inflate", snappy_inflate, -1);
     rb_define_singleton_method(rb_mSnappy, "valid?",  snappy_valid_p, 1);

data/ext/extconf.rb

@@ -1,37 +1,34 @@
 require 'mkmf'
 require 'fileutils'
 
-def patch_autogen
-  # s/libtoolize/glibtoolize/
-  File.write('autogen.sh', File.read('autogen.sh').gsub(/libtoolize/, 'glibtoolize'))
-end
-
-unless have_library 'snappy'
+unless pkg_config('libsnappy') || have_library('snappy')
   # build vendor/snappy
   pwd = File.dirname File.expand_path __FILE__
   dir = File.join pwd, '..', 'vendor', 'snappy'
 
-  Dir.chdir dir do
-    unless system './autogen.sh'
-      patch_autogen
-      raise '`autogen.sh` failed' unless system './autogen.sh'
+  Dir.chdir(dir) do
+    FileUtils.mkdir_p 'build'
+    Dir.chdir(File.join(dir, 'build')) do
+      `cmake .. -DCMAKE_BUILD_TYPE=Release`
     end
-    raise '`configure` failed'  unless system './configure --disable-option-checking --disable-dependency-tracking --disable-gtest --without-gflags'
   end
 
-  src = %w(
-   config.h
-   snappy-c.cc
-   snappy-c.h
-   snappy-internal.h
-   snappy-sinksource.cc
-   snappy-sinksource.h
-   snappy-stubs-internal.cc
-   snappy-stubs-internal.h
-   snappy-stubs-public.h
-   snappy.cc
-   snappy.h
-  ).map { |e| File.join dir, e }
+  src = %w[
+    config.h
+    snappy-stubs-public.h
+  ].map { |e| File.join dir, 'build', e }
+  FileUtils.cp src, pwd, :verbose => true
+  src = %w[
+    snappy-c.cc
+    snappy-c.h
+    snappy-internal.h
+    snappy-sinksource.cc
+    snappy-sinksource.h
+    snappy.cc
+    snappy.h
+    snappy-stubs-internal.cc
+    snappy-stubs-internal.h
+  ].map { |e| File.join dir, e }
   FileUtils.cp src, pwd, :verbose => true
 end
 

data/lib/snappy.rb

@@ -1,6 +1,8 @@
 # encoding: utf-8
+# frozen_string_literal: true
 
-require 'snappy-jars' if defined?(JRUBY_VERSION)
-require 'snappy_ext'
-require 'snappy/reader'
-require 'snappy/writer'
+require "snappy-jars" if defined?(JRUBY_VERSION)
+require "snappy_ext"
+require "snappy/reader"
+require "snappy/writer"
+require "snappy/hadoop"

data/lib/snappy/hadoop.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/hadoop/reader'
+require 'snappy/hadoop/writer'
+
+module Snappy
+  module Hadoop
+    def self.deflate(source, block_size = Snappy::Hadoop::Writer::DEFAULT_BLOCK_SIZE)
+      compressed_io = StringIO.new
+      writer = Snappy::Hadoop::Writer.new(compressed_io)
+      writer << source
+      writer.flush
+      compressed_io.string
+    end
+
+    def self.inflate(source)
+      Snappy::Hadoop::Reader.new(StringIO.new(source)).read
+    end
+  end
+end

data/lib/snappy/hadoop/reader.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Reader
+      attr_reader :io
+
+      def initialize(io)
+        @io = Snappy.set_encoding io
+        yield self if block_given?
+      end
+
+      def each
+        return to_enum unless block_given?
+
+        until @io.eof?
+          # Uncompressed size (32 bit integer, BE).
+          uncompressed_size = @io.read(4).unpack('N').first
+
+          uncompressed_block_io = StringIO.new
+          while uncompressed_block_io.size < uncompressed_size
+            # Compressed subblock size (32 bit integer, BE).
+            compressed_size = @io.read(4).unpack('N').first
+            uncompressed_block_io << Snappy.inflate(@io.read(compressed_size))
+          end
+
+          if uncompressed_block_io.size > uncompressed_size
+            raise RuntimeError("Invalid data: expected #{uncompressed_size} bytes, got #{Uncompressed.size}")
+          end
+
+          yield uncompressed_block_io.string
+        end
+      end
+
+      def read
+        buff = StringIO.new
+        each do |chunk|
+          buff << chunk
+        end
+        buff.string
+      end
+
+      def each_line(sep_string = $/)
+        return to_enum(:each_line, sep_string) unless block_given?
+
+        last = ""
+        each do |chunk|
+          chunk = last + chunk
+          lines = chunk.split(sep_string)
+          last = lines.pop
+          lines.each do |line|
+            yield line
+          end
+        end
+        yield last
+      end
+    end
+  end
+end

data/lib/snappy/hadoop/writer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Writer
+      DEFAULT_BLOCK_SIZE = 256 * 1024
+
+      attr_reader :io, :block_size
+
+      def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
+        @block_size = block_size
+        @buffer = String.new
+        @io = Snappy.set_encoding io
+        if block_given?
+          yield self
+          dump!
+        end
+      end
+
+      def <<(msg)
+        @buffer << msg.to_s
+        dump! if @buffer.size >= @block_size
+      end
+
+      alias_method :write, :<<
+
+      def dump!
+        offset = 0
+        while offset < @buffer.size
+          uncompressed = @buffer[offset, @block_size]
+          compressed = Snappy.deflate(uncompressed)
+
+          # Uncompressed size (32 bit integer, BE), compressed size (32 bit integer, BE), data.
+          @io << [uncompressed.size, compressed.size, compressed].pack("NNa#{compressed.size}")
+          offset += uncompressed.size
+        end
+
+        @io.flush
+        @buffer.clear
+      end
+
+      alias_method :flush, :dump!
+
+      def close
+        @io.close
+      end
+    end
+  end
+end