snappy 0.0.12-java → 0.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ad3c31a8a7d40bdd357a4604828f89212a4808af
-  data.tar.gz: ed9fd36c6e42df33b37203722d5f680545902fb4
+SHA256:
+  metadata.gz: 40598da197be3dd37207207288f7d07e864db1854075e05083aab32d0f36c2e7
+  data.tar.gz: 4e530705d88db960c3b0bd3474d33670ae0e79d55d0202770557fca817798efb
 SHA512:
-  metadata.gz: d326d3a46cd91ccd2ded284e65f2a0929cf7712355b83c7a839a3c3ed60e38dd39e5cea3badfb07aed8af944749488dc8cab9e6f4d1883f033fd4fb638e18cd2
-  data.tar.gz: 4dcc660ffffc348070ca7db7653baadcb2e44d583c3930b18b3dac0dfbc2bc8faa2bdc933df12a92d11973dcd84e10c2dc8e6a1a31c3164dad1558a1b2a812ac
+  metadata.gz: 27e1b4cc2d86dff4357130204a94c6212edd8999379be5dcd282ecdad59359ecefded8bc6bb122160683e2a005f8f1d8c229cc2e151c29eaaca0f6967bdd4901
+  data.tar.gz: 240cab9f029fdbe3132bfa76f846135b1d38d9661af81e353650a0674d62333a1025b8bbbcb291167ceca704d6ae10487cb78eb47009aac689af93542f5be474

data/.travis.yml

@@ -1,4 +1,31 @@
 language: ruby
+sudo: false
+cache:
+  bundler: true
+  directories:
+    - "$HOME/.ccache"
 rvm:
-  - 2.2.2
+  - 1.9.3
+  - 2.2
+  - 2.3
+  - 2.4
+  - 2.5
   - jruby-19mode
+env:
+  global:
+    - PATH="/usr/lib/ccache:$PATH"
+before_install:
+  - travis_retry gem install bundler --no-document -v 1.16.1
+addons:
+  apt:
+    packages:
+      - cmake
+deploy:
+  provider: rubygems
+  api_key:
+    secure: Jwk5pSUD9dUb34fO4dLamqhQw9U/t/5pHX7l5LKE4e1MDlD9fq/KB/DFpRTSfA0s8XUu+NZYtRO+6GZBEBY5UjZyPcUk5izaQzMWANP6QEMI4mYheSFUXTGkimV8MFxo2Fp7itZBM9SsLccDp/59nmbhBUMdr4OECZtd1Igjz63AtRdD7JgZwhC2XqBVbcxyx78VcZH8r4JEZzpcnZjjGN9YwQFubhiPs0V2uTMp/kGSWeP4U3T71MrvRmDlu8QJTsTRsvhv+zDatFySzs2SGsn25+cUCOqWnb5vcwtCbfj7trY+DLO3FqZOvQlvDfvyOy9LlLRpovhu6w+L42Z2ZN5U1Ovz0bMjk5WGLlXRxCLTRqXihvJrtFuL/7B2YSXcWPBNt5HWFwtapEJDm+CJoKpqa/WB7gEgO+uAocHbRqbg64rnPLODOEH0xGeRNSXBA3P5JeVNYhvpFdX8LclRFaVpfYDDenzRV9zpIppMXSq3Suzw/nZPBL9cuuIsBYG6hnijhZertWIRjZqw8D/1wEBd6VT7cDQ+PsNBGKhvPBVd5Nb7Zh9YjwCv4BQHB6XekiKi7mGFN4X/6eTzudRYxG7KyrYqk9AQ6KmEv5U/Xv+Wcar86O49LtNqYykYKLknC9CiUwTDp4lkVD8D3LoIzkL2/AeZyI6NQso8tymB0CM=
+  gem: snappy
+  on:
+    tags: true
+    repo: miyucy/snappy
+script: bundle exec rake && ./smoke.sh

data/Gemfile

@@ -1,4 +1,9 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in snappy.gemspec
-gemspec
+gemspec
+
+group :development do
+  gem 'rake'
+  gem 'minitest'
+end

data/README.md

@@ -4,13 +4,27 @@ see https://github.com/google/snappy
 
 ## Preparation
 
-Use libsnappy
 
-    $ brew install snappy
+### macOS
 
-Or
+```bash
+$ brew install snappy
+$ brew install autoconf automake libtool
+```
+
+### Ubuntu
+
+```bash
+$ apt-get install libsnappy-dev -y
+$ apt-get install libtool automake autoconf -y
+```
+
+### Alpine
 
-    $ brew install autoconf automake libtool
+```bash
+$ apk install snappy
+$ apk install build-base libexecinfo automake autoconf libtool
+```
 
 ## Installation
 
@@ -28,6 +42,7 @@ Or install it yourself as:
 
 ## Usage
 
+Snappy-java format
 ```ruby
 require 'snappy'
 
@@ -38,6 +53,15 @@ Snappy.inflate(source)
 # => Decompressed data
 ```
 
+Hadoop-snappy format
+```ruby
+Snappy::Hadoop.deflate(source)
+# => Compressed data
+
+Snappy::Hadoop.inflate(source)
+# => Decompressed data
+```
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -5,6 +5,7 @@ require "rbconfig"
 DLEXT = RbConfig::CONFIG['DLEXT']
 
 Rake::TestTask.new do |t|
+  t.pattern = 'test/**/test-*.rb'
   t.warning = true
   t.verbose = true
 end

data/ext/extconf.rb

@@ -1,37 +1,34 @@
 require 'mkmf'
 require 'fileutils'
 
-def patch_autogen
-  # s/libtoolize/glibtoolize/
-  File.write('autogen.sh', File.read('autogen.sh').gsub(/libtoolize/, 'glibtoolize'))
-end
-
-unless have_library 'snappy'
+unless pkg_config('libsnappy') || have_library('snappy')
   # build vendor/snappy
   pwd = File.dirname File.expand_path __FILE__
   dir = File.join pwd, '..', 'vendor', 'snappy'
 
-  Dir.chdir dir do
-    unless system './autogen.sh'
-      patch_autogen
-      raise '`autogen.sh` failed' unless system './autogen.sh'
+  Dir.chdir(dir) do
+    FileUtils.mkdir_p 'build'
+    Dir.chdir(File.join(dir, 'build')) do
+      `cmake .. -DCMAKE_BUILD_TYPE=Release`
     end
-    raise '`configure` failed'  unless system './configure --disable-option-checking --disable-dependency-tracking --disable-gtest --without-gflags'
   end
 
-  src = %w(
-   config.h
-   snappy-c.cc
-   snappy-c.h
-   snappy-internal.h
-   snappy-sinksource.cc
-   snappy-sinksource.h
-   snappy-stubs-internal.cc
-   snappy-stubs-internal.h
-   snappy-stubs-public.h
-   snappy.cc
-   snappy.h
-  ).map { |e| File.join dir, e }
+  src = %w[
+    config.h
+    snappy-stubs-public.h
+  ].map { |e| File.join dir, 'build', e }
+  FileUtils.cp src, pwd, :verbose => true
+  src = %w[
+    snappy-c.cc
+    snappy-c.h
+    snappy-internal.h
+    snappy-sinksource.cc
+    snappy-sinksource.h
+    snappy.cc
+    snappy.h
+    snappy-stubs-internal.cc
+    snappy-stubs-internal.h
+  ].map { |e| File.join dir, e }
   FileUtils.cp src, pwd, :verbose => true
 end
 

data/lib/snappy.rb

@@ -1,6 +1,8 @@
 # encoding: utf-8
+# frozen_string_literal: true
 
 require 'snappy-jars' if defined?(JRUBY_VERSION)
 require 'snappy_ext'
 require 'snappy/reader'
-require 'snappy/writer'
+require 'snappy/writer'
+require 'snappy/hadoop'

data/lib/snappy/hadoop.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/hadoop/reader'
+require 'snappy/hadoop/writer'
+
+module Snappy
+  module Hadoop
+    def self.deflate(source, block_size = Snappy::Hadoop::Writer::DEFAULT_BLOCK_SIZE)
+      compressed_io = StringIO.new
+      writer = Snappy::Hadoop::Writer.new(compressed_io)
+      writer << source
+      writer.flush
+      compressed_io.string
+    end
+
+    def self.inflate(source)
+      Snappy::Hadoop::Reader.new(StringIO.new(source)).read
+    end
+  end
+end

data/lib/snappy/hadoop/reader.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Reader
+      attr_reader :io
+
+      def initialize(io)
+        @io = Snappy.set_encoding io
+        yield self if block_given?
+      end
+
+      def each
+        until @io.eof?
+          # Uncompressed size (32 bit integer, BE).
+          uncompressed_size = @io.read(4).unpack('N').first
+
+          uncompressed_block_io = StringIO.new
+          while uncompressed_block_io.size < uncompressed_size
+            # Compressed subblock size (32 bit integer, BE).
+            compressed_size = @io.read(4).unpack('N').first
+            uncompressed_block_io << Snappy.inflate(@io.read(compressed_size))
+          end
+
+          if uncompressed_block_io.size > uncompressed_size
+            raise RuntimeError("Invalid data: expected #{uncompressed_size} bytes, got #{Uncompressed.size}")
+          end
+
+          yield uncompressed_block_io.string if block_given?
+        end
+      end
+
+      def read
+        buff = StringIO.new
+        each do |chunk|
+          buff << chunk
+        end
+        buff.string
+      end
+
+      def each_line(sep_string = $/)
+        last = ""
+        each do |chunk|
+          chunk = last + chunk
+          lines = chunk.split(sep_string)
+          last = lines.pop
+          lines.each do |line|
+            yield line if block_given?
+          end
+        end
+        yield last
+      end
+    end
+  end
+end

data/lib/snappy/hadoop/writer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Writer
+      DEFAULT_BLOCK_SIZE = 256 * 1024
+
+      attr_reader :io, :block_size
+
+      def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
+        @block_size = block_size
+        @buffer = String.new
+        @io = Snappy.set_encoding io
+        if block_given?
+          yield self
+          dump!
+        end
+      end
+
+      def <<(msg)
+        @buffer << msg.to_s
+        dump! if @buffer.size >= @block_size
+      end
+
+      alias_method :write, :<<
+
+      def dump!
+        offset = 0
+        while offset < @buffer.size
+          uncompressed = @buffer[offset, @block_size]
+          compressed = Snappy.deflate(uncompressed)
+
+          # Uncompressed size (32 bit integer, BE), compressed size (32 bit integer, BE), data.
+          @io << [uncompressed.size, compressed.size, compressed].pack("NNa#{compressed.size}")
+          offset += uncompressed.size
+        end
+
+        @io.flush
+        @buffer.clear
+      end
+
+      alias_method :flush, :dump!
+
+      def close
+        @io.close
+      end
+    end
+  end
+end

data/lib/snappy/reader.rb

@@ -1,10 +1,14 @@
+# frozen_string_literal: true
+
+require 'snappy/shim'
+require 'snappy/writer'
+
 module Snappy
   class Reader
     attr_reader :io, :magic, :default_version, :minimum_compatible_version
 
     def initialize(io)
-      @io = io
-      @io.set_encoding Encoding::ASCII_8BIT unless RUBY_VERSION =~ /^1\.8/
+      @io = Snappy.set_encoding io
       read_header!
       yield self if block_given?
     end
@@ -15,20 +19,20 @@ module Snappy
           size = @io.read(4).unpack('N').first
           yield Snappy.inflate(@io.read(size)) if block_given?
         else
-          yield Snappy.inflate @io.read if block_given?
+          yield Snappy.inflate(@io.read) if block_given?
         end
       end
     end
 
     def read
-      @buff = StringIO.new
+      buff = StringIO.new
       each do |chunk|
-        @buff << chunk
+        buff << chunk
       end
-      @buff.string
+      buff.string
     end
 
-    def each_line(sep_string=$/)
+    def each_line(sep_string = $/)
       last = ""
       each do |chunk|
         chunk = last + chunk

data/lib/snappy/shim.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Snappy
+  module_function
+
+  if RUBY_VERSION[0..2] == '1.8'
+    def set_encoding(io)
+      io
+    end
+
+    def b(str)
+      str
+    end
+  else
+    def set_encoding(io)
+      io.set_encoding Encoding::ASCII_8BIT
+      io
+    end
+
+    if ::String.instance_methods.include? :b
+      def b(str)
+        str.b
+      end
+    else
+      def b(str)
+        str.force_encoding Encoding::ASCII_8BIT
+      end
+    end
+  end
+end

data/lib/snappy/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Snappy
-  VERSION = "0.0.12"
+  VERSION = "0.1.0"
 end

data/lib/snappy/writer.rb

@@ -1,10 +1,10 @@
+# frozen_string_literal: true
+
+require 'snappy/shim'
+
 module Snappy
   class Writer
-    if RUBY_VERSION[0..2] == '1.8'
-      MAGIC = "\x82SNAPPY\x0"
-    else
-      MAGIC = "\x82SNAPPY\x0".force_encoding Encoding::ASCII_8BIT
-    end
+    MAGIC = Snappy.b("\x82SNAPPY\x0")
     DEFAULT_VERSION = 1
     MINIMUM_COMPATIBLE_VERSION = 1
     DEFAULT_BLOCK_SIZE = 32 * 1024
@@ -13,9 +13,8 @@ module Snappy
 
     def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
       @block_size = block_size
-      @buffer = ""
-      @io = io
-      @io.set_encoding Encoding::ASCII_8BIT unless RUBY_VERSION =~ /^1\.8/
+      @buffer = String.new
+      @io = Snappy.set_encoding io
       write_header!
       if block_given?
         yield self
@@ -28,15 +27,21 @@ module Snappy
       dump! if @buffer.size > @block_size
     end
 
+    alias_method :write, :<<
+
     def dump!
       compressed = Snappy.deflate(@buffer)
       @io << [compressed.size, compressed].pack("Na#{compressed.size}")
       @io.flush
-      @buffer = ""
+      @buffer.clear
     end
 
     alias_method :flush, :dump!
 
+    def close
+      @io.close
+    end
+
     private
 
     def write_header!