snappy 0.0.17 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 69133bb0d8adece5bc23fcf0db6e66180c58b04c
-  data.tar.gz: 410ae9d85c9260000a0b816efe4954a92f4188d9
+SHA256:
+  metadata.gz: 4866cfb8c9b7b1e16e5d6c8ba7044d8e003d5a03274575a09d60227bbdf4c603
+  data.tar.gz: dfd8a8d4cdb51bce5a988b55c61940d9717fb95a9f9377f32db93708d43805ce
 SHA512:
-  metadata.gz: 18927ee51e2ddca5bc23c1fa80d207f8b9c2ae9450085606153233dd000c1c1f72bf14f43a2248d1290328bfde776760e68800d10526447b96cd975824b7528c
-  data.tar.gz: 8f9ed859c2c472a408425d1981c63a92c0c7349d1e1a4426b422dd087d89f70167ed0c80e009f581c0f348d65afa5ef017cb2ee96f3620049f921f0260c09f36
+  metadata.gz: 1f3f35f369f73dee25075f72dc48693833d084d6bc6844243ef82585f41255bbf6bedaece227b05aa3ecc8bd9f9db2a13c98ea9ee1f9894204a719b9e0e0f57c
+  data.tar.gz: 36af767eded664e7bb25e120e807897d47a6d6c5a86d6f655970030a20636b14143ae8e1df84d9032aa175987b2d97a21c31d1c048f0422f7da00079bce38d23

data/.dockerignore

@@ -0,0 +1,2 @@
+vendor/bundle
+Dockerfile

data/.github/workflows/main.yml

@@ -0,0 +1,34 @@
+name: Ruby
+
+on: [pull_request]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.5, 2.6, 2.7, 3.0, 3.1, jruby]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Run the default task
+      run: |
+        bundle exec rake clobber clean test build
+        gem install --no-document "$(ls pkg/snappy-*.gem)"
+        cat <<EOF | ruby
+        require "snappy"
+        if Snappy.inflate(Snappy.deflate(File.read("README.md"))) == File.read("README.md")
+          puts "OK"
+          exit 0
+        else
+          puts "NG"
+          exit 0
+        end
+        EOF

data/.github/workflows/publish.yml

@@ -0,0 +1,34 @@
+name: Publish Gem
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.7, jruby]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Run release task
+      run: |
+        mkdir -p ~/.gem
+        cat << EOF > ~/.gem/credentials
+        ---
+        :github: Bearer ${{secrets.GITHUB_TOKEN}}
+        :rubygems_api_key: ${{secrets.RUBYGEMS_API_KEY}}
+        EOF
+        chmod 600 ~/.gem/credentials
+        bundle exec rake release[remote]
+        rm -f ~/.gem/credentials

data/.gitignore

@@ -21,4 +21,5 @@ ext/config.h
 ext/*.o
 ext/*.log
 ext/snappy*
-ext/java/build
+ext/java/build
+vendor/bundle

data/.gitmodules

@@ -1,3 +1,3 @@
 [submodule "vendor/snappy"]
 	path = vendor/snappy
-	url = git://github.com/google/snappy.git
+	url = https://github.com/google/snappy.git

data/Dockerfile

@@ -0,0 +1,13 @@
+# syntax = docker/dockerfile:labs
+ARG RUBY_VERSION=3.1
+FROM ruby:${RUBY_VERSION}
+
+ENV BUNDLE_JOBS=4
+
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt <<EOF
+apt-get update
+apt-get install -y \
+  cmake
+EOF
+
+WORKDIR /app

data/Gemfile

@@ -2,3 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in snappy.gemspec
 gemspec
+
+gem "rake", "~> 13.0"
+gem "test-unit", "~> 3.0"
+gem "test-unit-rr"

data/README.md

@@ -1,16 +1,30 @@
-# Snappy ![](https://travis-ci.org/miyucy/snappy.svg?branch=master)
+# Snappy
 
 see https://github.com/google/snappy
 
 ## Preparation
 
-Use libsnappy
 
-    $ brew install snappy
+### macOS
 
-Or
+```bash
+$ brew install snappy
+$ brew install autoconf automake libtool
+```
+
+### Ubuntu
+
+```bash
+$ apt-get install libsnappy-dev -y
+$ apt-get install libtool automake autoconf -y
+```
+
+### Alpine
 
-    $ brew install autoconf automake libtool
+```bash
+$ apk install snappy
+$ apk install build-base libexecinfo automake autoconf libtool
+```
 
 ## Installation
 
@@ -28,6 +42,7 @@ Or install it yourself as:
 
 ## Usage
 
+Snappy-java format
 ```ruby
 require 'snappy'
 
@@ -38,6 +53,15 @@ Snappy.inflate(source)
 # => Decompressed data
 ```
 
+Hadoop-snappy format
+```ruby
+Snappy::Hadoop.deflate(source)
+# => Compressed data
+
+Snappy::Hadoop.inflate(source)
+# => Decompressed data
+```
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -2,48 +2,51 @@ require "bundler/setup"
 require "bundler/gem_tasks"
 require "rake/testtask"
 require "rbconfig"
-DLEXT = RbConfig::CONFIG['DLEXT']
+DLEXT = RbConfig::CONFIG["DLEXT"]
 
-Rake::TestTask.new do |t|
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
   t.warning = true
   t.verbose = true
 end
 
 if defined?(JRUBY_VERSION)
-  require 'ant'
+  require "ant"
 
-  directory 'ext/java/build'
+  directory "ext/java/build"
 
-  task :setup => 'ext/java/build' do
-    ant.property name: 'src.dir', value: 'ext/java/src'
-    ant.property name: 'build.dir', value: 'ext/java/build'
+  task setup: "ext/java/build" do
+    ant.property name: "src.dir", value: "ext/java/src"
+    ant.property name: "build.dir", value: "ext/java/build"
 
-    ant.path id: 'compile.class.path' do
-      pathelement location: File.join(RbConfig::CONFIG['prefix'], 'lib', 'jruby.jar')
-      $LOAD_PATH.flat_map { |path| Dir[File.join(path, '**', '*.jar')] }.each do |jar|
+    ant.path id: "compile.class.path" do
+      pathelement location: File.join(RbConfig::CONFIG["prefix"], "lib", "jruby.jar")
+      $LOAD_PATH.flat_map { |path| Dir[File.join(path, "**", "*.jar")] }.each do |jar|
         pathelement location: jar
       end
     end
   end
 
-  desc 'Compile the extension'
-  task :compile => :setup do
-    ant.javac destdir: '${build.dir}', includeantruntime: 'no', target: '1.6', source: '1.6', debug: 'on' do
-      classpath refid: 'compile.class.path'
-      src { pathelement location: '${src.dir}' }
+  desc "Compile the extension"
+  task compile: :setup do
+    ant.javac destdir: "${build.dir}", includeantruntime: "no", target: "1.6", source: "1.6", debug: "on" do
+      classpath refid: "compile.class.path"
+      src { pathelement location: "${src.dir}" }
     end
   end
 
-  desc 'Package the jar'
-  file 'lib/snappy_ext.jar' => :compile do |t|
-    ant.jar destfile: 'lib/snappy_ext.jar', basedir: '${build.dir}' do
-      ant.fileset dir: '${build.dir}', includes: 'snappy/*.class'
-      ant.fileset dir: '${build.dir}', includes: 'SnappyExtLibraryService.class'
+  desc "Package the jar"
+  file "lib/snappy_ext.jar" => :compile do |t|
+    ant.jar destfile: "lib/snappy_ext.jar", basedir: "${build.dir}" do
+      ant.fileset dir: "${build.dir}", includes: "snappy/*.class"
+      ant.fileset dir: "${build.dir}", includes: "SnappyExtLibraryService.class"
     end
   end
 
-  task :test => 'lib/snappy_ext.jar'
-  task :build => [:clean, 'lib/snappy_ext.jar']
+  task test: "lib/snappy_ext.jar"
+  task build: [:clean, "lib/snappy_ext.jar"]
 else
   file "ext/snappy_ext.#{DLEXT}" => Dir.glob("ext/*{.rb,.c}") do
     Dir.chdir("ext") do
@@ -53,15 +56,15 @@ else
     cp "ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}"
   end
 
-
-  task :test => "ext/snappy_ext.#{DLEXT}"
+  task test: "ext/snappy_ext.#{DLEXT}"
 end
 
-desc 'Clean up build artifacts'
+desc "Clean up build artifacts"
 task :clean do
-  rm_rf 'ext/java/build'
-  rm_rf 'lib/snappy_ext.jar'
-  rm_rf(["ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}", 'ext/mkmf.log', 'ext/config.h', 'ext/api.o', 'ext/Makefile', 'ext/snappy.cc', 'ext/snappy.h', 'ext/snappy.o'] + Dir['ext/snappy-*'])
+  rm_rf "ext/java/build"
+  rm_rf "lib/snappy_ext.jar"
+  rm_rf(["ext/snappy_ext.#{DLEXT}", "lib/snappy_ext.#{DLEXT}", "ext/mkmf.log", "ext/config.h", "ext/api.o",
+         "ext/Makefile", "ext/snappy.cc", "ext/snappy.h", "ext/snappy.o"] + Dir["ext/snappy-*"])
 end
 
-task :default => :test
+task default: :test

data/ext/api.c

@@ -1,7 +1,6 @@
 #include "ruby.h"
 #include "snappy-c.h"
 
-static VALUE rb_mSnappy;
 static VALUE rb_eSnappy;
 
 static VALUE
@@ -92,10 +91,16 @@ snappy_valid_p(VALUE self, VALUE str)
 
 void Init_snappy_ext()
 {
+#if HAVE_RB_EXT_RACTOR_SAFE
+    rb_ext_ractor_safe(true);
+#endif
+
+    VALUE rb_mSnappy;
     VALUE rb_mSnappy_singleton;
 
     rb_mSnappy = rb_define_module("Snappy");
     rb_eSnappy = rb_define_class_under(rb_mSnappy, "Error", rb_eStandardError);
+    rb_global_variable(&rb_eSnappy);
     rb_define_singleton_method(rb_mSnappy, "deflate", snappy_deflate, -1);
     rb_define_singleton_method(rb_mSnappy, "inflate", snappy_inflate, -1);
     rb_define_singleton_method(rb_mSnappy, "valid?",  snappy_valid_p, 1);

data/ext/extconf.rb

@@ -1,29 +1,36 @@
 require 'mkmf'
 require 'fileutils'
 
+$CXXFLAGS += " -std=c++11 "
+
 unless pkg_config('libsnappy') || have_library('snappy')
   # build vendor/snappy
   pwd = File.dirname File.expand_path __FILE__
   dir = File.join pwd, '..', 'vendor', 'snappy'
 
-  Dir.chdir dir do
-    raise '`autogen.sh` failed' unless system './autogen.sh'
-    raise '`configure` failed'  unless system './configure --disable-option-checking --disable-dependency-tracking --disable-gtest --without-gflags'
+  Dir.chdir(dir) do
+    FileUtils.mkdir_p 'build'
+    Dir.chdir(File.join(dir, 'build')) do
+      `cmake .. -DCMAKE_BUILD_TYPE=Release`
+    end
   end
 
-  src = %w(
-   config.h
-   snappy-c.cc
-   snappy-c.h
-   snappy-internal.h
-   snappy-sinksource.cc
-   snappy-sinksource.h
-   snappy-stubs-internal.cc
-   snappy-stubs-internal.h
-   snappy-stubs-public.h
-   snappy.cc
-   snappy.h
-  ).map { |e| File.join dir, e }
+  src = %w[
+    config.h
+    snappy-stubs-public.h
+  ].map { |e| File.join dir, 'build', e }
+  FileUtils.cp src, pwd, :verbose => true
+  src = %w[
+    snappy-c.cc
+    snappy-c.h
+    snappy-internal.h
+    snappy-sinksource.cc
+    snappy-sinksource.h
+    snappy.cc
+    snappy.h
+    snappy-stubs-internal.cc
+    snappy-stubs-internal.h
+  ].map { |e| File.join dir, e }
   FileUtils.cp src, pwd, :verbose => true
 end
 

data/lib/snappy/hadoop/reader.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Reader
+      attr_reader :io
+
+      def initialize(io)
+        @io = Snappy.set_encoding io
+        yield self if block_given?
+      end
+
+      def each
+        return to_enum unless block_given?
+
+        until @io.eof?
+          # Uncompressed size (32 bit integer, BE).
+          uncompressed_size = @io.read(4).unpack('N').first
+
+          uncompressed_block_io = StringIO.new
+          while uncompressed_block_io.size < uncompressed_size
+            # Compressed subblock size (32 bit integer, BE).
+            compressed_size = @io.read(4).unpack('N').first
+            uncompressed_block_io << Snappy.inflate(@io.read(compressed_size))
+          end
+
+          if uncompressed_block_io.size > uncompressed_size
+            raise RuntimeError("Invalid data: expected #{uncompressed_size} bytes, got #{Uncompressed.size}")
+          end
+
+          yield uncompressed_block_io.string
+        end
+      end
+
+      def read
+        buff = StringIO.new
+        each do |chunk|
+          buff << chunk
+        end
+        buff.string
+      end
+
+      def each_line(sep_string = $/)
+        return to_enum(:each_line, sep_string) unless block_given?
+
+        last = ""
+        each do |chunk|
+          chunk = last + chunk
+          lines = chunk.split(sep_string)
+          last = lines.pop
+          lines.each do |line|
+            yield line
+          end
+        end
+        yield last
+      end
+    end
+  end
+end

data/lib/snappy/hadoop/writer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Writer
+      DEFAULT_BLOCK_SIZE = 256 * 1024
+
+      attr_reader :io, :block_size
+
+      def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
+        @block_size = block_size
+        @buffer = String.new
+        @io = Snappy.set_encoding io
+        if block_given?
+          yield self
+          dump!
+        end
+      end
+
+      def <<(msg)
+        @buffer << msg.to_s
+        dump! if @buffer.size >= @block_size
+      end
+
+      alias_method :write, :<<
+
+      def dump!
+        offset = 0
+        while offset < @buffer.size
+          uncompressed = @buffer[offset, @block_size]
+          compressed = Snappy.deflate(uncompressed)
+
+          # Uncompressed size (32 bit integer, BE), compressed size (32 bit integer, BE), data.
+          @io << [uncompressed.size, compressed.size, compressed].pack("NNa#{compressed.size}")
+          offset += uncompressed.size
+        end
+
+        @io.flush
+        @buffer.clear
+      end
+
+      alias_method :flush, :dump!
+
+      def close
+        @io.close
+      end
+    end
+  end
+end

data/lib/snappy/hadoop.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/hadoop/reader'
+require 'snappy/hadoop/writer'
+
+module Snappy
+  module Hadoop
+    def self.deflate(source, block_size = Snappy::Hadoop::Writer::DEFAULT_BLOCK_SIZE)
+      compressed_io = StringIO.new
+      writer = Snappy::Hadoop::Writer.new(compressed_io)
+      writer << source
+      writer.flush
+      compressed_io.string
+    end
+
+    def self.inflate(source)
+      Snappy::Hadoop::Reader.new(StringIO.new(source)).read
+    end
+  end
+end

data/lib/snappy/reader.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'snappy/shim'
-require 'snappy/writer'
+require "snappy/shim"
+require "snappy/writer"
 
 module Snappy
   class Reader
@@ -14,32 +14,36 @@ module Snappy
     end
 
     def each
+      return to_enum unless block_given?
+
       until @io.eof?
         if @chunked
-          size = @io.read(4).unpack('N').first
-          yield Snappy.inflate(@io.read(size)) if block_given?
+          size = @io.read(4).unpack1("N")
+          yield Snappy.inflate(@io.read(size))
         else
-          yield Snappy.inflate @io.read if block_given?
+          yield Snappy.inflate(@io.read)
         end
       end
     end
 
     def read
-      @buff = StringIO.new
+      buff = StringIO.new
       each do |chunk|
-        @buff << chunk
+        buff << chunk
       end
-      @buff.string
+      buff.string
     end
 
     def each_line(sep_string = $/)
+      return to_enum(:each_line, sep_string) unless block_given?
+
       last = ""
       each do |chunk|
         chunk = last + chunk
         lines = chunk.split(sep_string)
         last = lines.pop
         lines.each do |line|
-          yield line if block_given?
+          yield line
         end
       end
       yield last
@@ -50,7 +54,7 @@ module Snappy
     def read_header!
       header = @io.read Snappy::Writer::MAGIC.length
       if header.length == Snappy::Writer::MAGIC.length && header == Snappy::Writer::MAGIC
-        @magic, @default_version, @minimum_compatible_version = header, *@io.read(8).unpack('NN')
+        @magic, @default_version, @minimum_compatible_version = header, *@io.read(8).unpack("NN")
         @chunked = true
       else
         @io.rewind

data/lib/snappy/shim.rb

@@ -3,7 +3,7 @@
 module Snappy
   module_function
 
-  if RUBY_VERSION[0..2] == '1.8'
+  if RUBY_VERSION[0..2] == "1.8"
     def set_encoding(io)
       io
     end

data/lib/snappy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Snappy
-  VERSION = "0.0.17"
+  VERSION = "0.3.0"
 end

data/lib/snappy.rb

@@ -1,7 +1,8 @@
 # encoding: utf-8
 # frozen_string_literal: true
 
-require 'snappy-jars' if defined?(JRUBY_VERSION)
-require 'snappy_ext'
-require 'snappy/reader'
-require 'snappy/writer'
+require "snappy-jars" if defined?(JRUBY_VERSION)
+require "snappy_ext"
+require "snappy/reader"
+require "snappy/writer"
+require "snappy/hadoop"

data/snappy.gemspec

@@ -1,36 +1,36 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'snappy/version'
+require "snappy/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "snappy"
   spec.version       = Snappy::VERSION
   spec.authors       = ["miyucy"]
   spec.email         = ["fistfvck@gmail.com"]
-  spec.description   = %q{libsnappy binding for Ruby}
-  spec.summary       = %q{libsnappy binding for Ruby}
+  spec.description   = "libsnappy binding for Ruby"
+  spec.summary       = "libsnappy binding for Ruby"
   spec.homepage      = "http://github.com/miyucy/snappy"
   spec.license       = "MIT"
 
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+
   spec.test_files    = `git ls-files -z -- test`.split("\x0")
   spec.files         = `git ls-files -z`.split("\x0")
   spec.files        -= spec.test_files
-  spec.files        -= ['vendor/snappy']
-  spec.files        += Dir['vendor/snappy/**/*'].reject { |e| e.start_with? 'vendor/snappy/testdata' }
+  spec.files        -= ["vendor/snappy"]
+  spec.files        += Dir["vendor/snappy/**/*"].reject { |e| e.start_with? "vendor/snappy/testdata" }
 
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
   if defined?(JRUBY_VERSION)
     spec.files += %w[lib/snappy_ext.jar]
-    spec.platform = 'java'
-    spec.add_dependency 'snappy-jars', '~> 1.1.0'
+    spec.platform = "java"
+    spec.add_dependency "snappy-jars", "~> 1.1.0"
   else
     spec.extensions = ["ext/extconf.rb"]
   end
-
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "minitest"
 end