snappy 0.0.13 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f767827959156bc4efe3531874c3cead0599d10a
-  data.tar.gz: 5c08643b387f25d4b82479b359572eedd377db38
+SHA256:
+  metadata.gz: 3b633f54dc239a8a5a4ef4df91fd0a23a100298ca83192d30096490f6174670b
+  data.tar.gz: 7caed929e615860f65dc0a25568cbe1c4f01ed377fbbc96de457acffb9dcc349
 SHA512:
-  metadata.gz: 40e63dafa6fad90a620169ee3d494d9b9052d07a6f3b81bcd76368b3dba195f50b1b9ebf7e9831ca584e08804a0993d45b4e904f14c25c91c9c5c5772e086792
-  data.tar.gz: 09e4d31244fc7d5e5182dc19373de522224f825b54f0c78b73d96dd426677a4714418bea063b57c1b8da2c5e63f06221cde9dea34459ecf884bd3c90a0672873
+  metadata.gz: 6c51f97d066a0bc256498157c9e9a2dab2da1191f310da24d0b54531de6ceb4db09471a4d8738180e20ca06f8e7147440ded43c6ad6baf1599f626c751a9086a
+  data.tar.gz: 8d8b5494df7e5858ebe77c2a86845257409cd71d793f8cd7c2123f9f784e75fb0d66294ce87ddd5fc5140b4702a135e8a644b5a07308414a9fc747fb6111658a

data/.travis.yml

@@ -1,4 +1,31 @@
 language: ruby
+sudo: false
+cache:
+  bundler: true
+  directories:
+    - "$HOME/.ccache"
 rvm:
-  - 2.2.2
+  - 1.9.3
+  - 2.2
+  - 2.3
+  - 2.4
+  - 2.5
   - jruby-19mode
+env:
+  global:
+    - PATH="/usr/lib/ccache:$PATH"
+before_install:
+  - travis_retry gem install bundler --no-document -v 1.16.1
+addons:
+  apt:
+    packages:
+      - cmake
+deploy:
+  provider: rubygems
+  api_key:
+    secure: Jwk5pSUD9dUb34fO4dLamqhQw9U/t/5pHX7l5LKE4e1MDlD9fq/KB/DFpRTSfA0s8XUu+NZYtRO+6GZBEBY5UjZyPcUk5izaQzMWANP6QEMI4mYheSFUXTGkimV8MFxo2Fp7itZBM9SsLccDp/59nmbhBUMdr4OECZtd1Igjz63AtRdD7JgZwhC2XqBVbcxyx78VcZH8r4JEZzpcnZjjGN9YwQFubhiPs0V2uTMp/kGSWeP4U3T71MrvRmDlu8QJTsTRsvhv+zDatFySzs2SGsn25+cUCOqWnb5vcwtCbfj7trY+DLO3FqZOvQlvDfvyOy9LlLRpovhu6w+L42Z2ZN5U1Ovz0bMjk5WGLlXRxCLTRqXihvJrtFuL/7B2YSXcWPBNt5HWFwtapEJDm+CJoKpqa/WB7gEgO+uAocHbRqbg64rnPLODOEH0xGeRNSXBA3P5JeVNYhvpFdX8LclRFaVpfYDDenzRV9zpIppMXSq3Suzw/nZPBL9cuuIsBYG6hnijhZertWIRjZqw8D/1wEBd6VT7cDQ+PsNBGKhvPBVd5Nb7Zh9YjwCv4BQHB6XekiKi7mGFN4X/6eTzudRYxG7KyrYqk9AQ6KmEv5U/Xv+Wcar86O49LtNqYykYKLknC9CiUwTDp4lkVD8D3LoIzkL2/AeZyI6NQso8tymB0CM=
+  gem: snappy
+  on:
+    tags: true
+    repo: miyucy/snappy
+script: bundle exec rake && ./smoke.sh

data/Gemfile

@@ -1,4 +1,9 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in snappy.gemspec
-gemspec
+gemspec
+
+group :development do
+  gem 'rake'
+  gem 'minitest'
+end

data/README.md

@@ -4,13 +4,27 @@ see https://github.com/google/snappy
 
 ## Preparation
 
-Use libsnappy
 
-    $ brew install snappy
+### macOS
 
-Or
+```bash
+$ brew install snappy
+$ brew install autoconf automake libtool
+```
+
+### Ubuntu
+
+```bash
+$ apt-get install libsnappy-dev -y
+$ apt-get install libtool automake autoconf -y
+```
+
+### Alpine
 
-    $ brew install autoconf automake libtool
+```bash
+$ apk install snappy
+$ apk install build-base libexecinfo automake autoconf libtool
+```
 
 ## Installation
 
@@ -28,6 +42,7 @@ Or install it yourself as:
 
 ## Usage
 
+Snappy-java format
 ```ruby
 require 'snappy'
 
@@ -38,6 +53,15 @@ Snappy.inflate(source)
 # => Decompressed data
 ```
 
+Hadoop-snappy format
+```ruby
+Snappy::Hadoop.deflate(source)
+# => Compressed data
+
+Snappy::Hadoop.inflate(source)
+# => Decompressed data
+```
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -5,6 +5,7 @@ require "rbconfig"
 DLEXT = RbConfig::CONFIG['DLEXT']
 
 Rake::TestTask.new do |t|
+  t.pattern = 'test/**/test-*.rb'
   t.warning = true
   t.verbose = true
 end

data/ext/extconf.rb

@@ -1,37 +1,34 @@
 require 'mkmf'
 require 'fileutils'
 
-def patch_autogen
-  # s/libtoolize/glibtoolize/
-  File.write('autogen.sh', File.read('autogen.sh').gsub(/libtoolize/, 'glibtoolize'))
-end
-
-unless have_library 'snappy'
+unless pkg_config('libsnappy') || have_library('snappy')
   # build vendor/snappy
   pwd = File.dirname File.expand_path __FILE__
   dir = File.join pwd, '..', 'vendor', 'snappy'
 
-  Dir.chdir dir do
-    unless system './autogen.sh'
-      patch_autogen
-      raise '`autogen.sh` failed' unless system './autogen.sh'
+  Dir.chdir(dir) do
+    FileUtils.mkdir_p 'build'
+    Dir.chdir(File.join(dir, 'build')) do
+      `cmake .. -DCMAKE_BUILD_TYPE=Release`
     end
-    raise '`configure` failed'  unless system './configure --disable-option-checking --disable-dependency-tracking --disable-gtest --without-gflags'
   end
 
-  src = %w(
-   config.h
-   snappy-c.cc
-   snappy-c.h
-   snappy-internal.h
-   snappy-sinksource.cc
-   snappy-sinksource.h
-   snappy-stubs-internal.cc
-   snappy-stubs-internal.h
-   snappy-stubs-public.h
-   snappy.cc
-   snappy.h
-  ).map { |e| File.join dir, e }
+  src = %w[
+    config.h
+    snappy-stubs-public.h
+  ].map { |e| File.join dir, 'build', e }
+  FileUtils.cp src, pwd, :verbose => true
+  src = %w[
+    snappy-c.cc
+    snappy-c.h
+    snappy-internal.h
+    snappy-sinksource.cc
+    snappy-sinksource.h
+    snappy.cc
+    snappy.h
+    snappy-stubs-internal.cc
+    snappy-stubs-internal.h
+  ].map { |e| File.join dir, e }
   FileUtils.cp src, pwd, :verbose => true
 end
 

data/lib/snappy.rb

@@ -1,6 +1,8 @@
 # encoding: utf-8
+# frozen_string_literal: true
 
 require 'snappy-jars' if defined?(JRUBY_VERSION)
 require 'snappy_ext'
 require 'snappy/reader'
-require 'snappy/writer'
+require 'snappy/writer'
+require 'snappy/hadoop'

data/lib/snappy/hadoop.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/hadoop/reader'
+require 'snappy/hadoop/writer'
+
+module Snappy
+  module Hadoop
+    def self.deflate(source, block_size = Snappy::Hadoop::Writer::DEFAULT_BLOCK_SIZE)
+      compressed_io = StringIO.new
+      writer = Snappy::Hadoop::Writer.new(compressed_io)
+      writer << source
+      writer.flush
+      compressed_io.string
+    end
+
+    def self.inflate(source)
+      Snappy::Hadoop::Reader.new(StringIO.new(source)).read
+    end
+  end
+end

data/lib/snappy/hadoop/reader.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Reader
+      attr_reader :io
+
+      def initialize(io)
+        @io = Snappy.set_encoding io
+        yield self if block_given?
+      end
+
+      def each
+        until @io.eof?
+          # Uncompressed size (32 bit integer, BE).
+          uncompressed_size = @io.read(4).unpack('N').first
+
+          uncompressed_block_io = StringIO.new
+          while uncompressed_block_io.size < uncompressed_size
+            # Compressed subblock size (32 bit integer, BE).
+            compressed_size = @io.read(4).unpack('N').first
+            uncompressed_block_io << Snappy.inflate(@io.read(compressed_size))
+          end
+
+          if uncompressed_block_io.size > uncompressed_size
+            raise RuntimeError("Invalid data: expected #{uncompressed_size} bytes, got #{Uncompressed.size}")
+          end
+
+          yield uncompressed_block_io.string if block_given?
+        end
+      end
+
+      def read
+        buff = StringIO.new
+        each do |chunk|
+          buff << chunk
+        end
+        buff.string
+      end
+
+      def each_line(sep_string = $/)
+        last = ""
+        each do |chunk|
+          chunk = last + chunk
+          lines = chunk.split(sep_string)
+          last = lines.pop
+          lines.each do |line|
+            yield line if block_given?
+          end
+        end
+        yield last
+      end
+    end
+  end
+end

data/lib/snappy/hadoop/writer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Writer
+      DEFAULT_BLOCK_SIZE = 256 * 1024
+
+      attr_reader :io, :block_size
+
+      def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
+        @block_size = block_size
+        @buffer = String.new
+        @io = Snappy.set_encoding io
+        if block_given?
+          yield self
+          dump!
+        end
+      end
+
+      def <<(msg)
+        @buffer << msg.to_s
+        dump! if @buffer.size >= @block_size
+      end
+
+      alias_method :write, :<<
+
+      def dump!
+        offset = 0
+        while offset < @buffer.size
+          uncompressed = @buffer[offset, @block_size]
+          compressed = Snappy.deflate(uncompressed)
+
+          # Uncompressed size (32 bit integer, BE), compressed size (32 bit integer, BE), data.
+          @io << [uncompressed.size, compressed.size, compressed].pack("NNa#{compressed.size}")
+          offset += uncompressed.size
+        end
+
+        @io.flush
+        @buffer.clear
+      end
+
+      alias_method :flush, :dump!
+
+      def close
+        @io.close
+      end
+    end
+  end
+end

data/lib/snappy/reader.rb

@@ -1,10 +1,14 @@
+# frozen_string_literal: true
+
+require 'snappy/shim'
+require 'snappy/writer'
+
 module Snappy
   class Reader
     attr_reader :io, :magic, :default_version, :minimum_compatible_version
 
     def initialize(io)
-      @io = io
-      @io.set_encoding Encoding::ASCII_8BIT unless RUBY_VERSION =~ /^1\.8/
+      @io = Snappy.set_encoding io
       read_header!
       yield self if block_given?
     end
@@ -15,20 +19,20 @@ module Snappy
           size = @io.read(4).unpack('N').first
           yield Snappy.inflate(@io.read(size)) if block_given?
         else
-          yield Snappy.inflate @io.read if block_given?
+          yield Snappy.inflate(@io.read) if block_given?
         end
       end
     end
 
     def read
-      @buff = StringIO.new
+      buff = StringIO.new
       each do |chunk|
-        @buff << chunk
+        buff << chunk
       end
-      @buff.string
+      buff.string
     end
 
-    def each_line(sep_string=$/)
+    def each_line(sep_string = $/)
       last = ""
       each do |chunk|
         chunk = last + chunk

data/lib/snappy/shim.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Snappy
+  module_function
+
+  if RUBY_VERSION[0..2] == '1.8'
+    def set_encoding(io)
+      io
+    end
+
+    def b(str)
+      str
+    end
+  else
+    def set_encoding(io)
+      io.set_encoding Encoding::ASCII_8BIT
+      io
+    end
+
+    if ::String.instance_methods.include? :b
+      def b(str)
+        str.b
+      end
+    else
+      def b(str)
+        str.force_encoding Encoding::ASCII_8BIT
+      end
+    end
+  end
+end

data/lib/snappy/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Snappy
-  VERSION = "0.0.13"
+  VERSION = "0.1.0"
 end

data/lib/snappy/writer.rb

@@ -1,10 +1,10 @@
+# frozen_string_literal: true
+
+require 'snappy/shim'
+
 module Snappy
   class Writer
-    if RUBY_VERSION[0..2] == '1.8'
-      MAGIC = "\x82SNAPPY\x0"
-    else
-      MAGIC = "\x82SNAPPY\x0".force_encoding Encoding::ASCII_8BIT
-    end
+    MAGIC = Snappy.b("\x82SNAPPY\x0")
     DEFAULT_VERSION = 1
     MINIMUM_COMPATIBLE_VERSION = 1
     DEFAULT_BLOCK_SIZE = 32 * 1024
@@ -13,9 +13,8 @@ module Snappy
 
     def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
       @block_size = block_size
-      @buffer = ""
-      @io = io
-      @io.set_encoding Encoding::ASCII_8BIT unless RUBY_VERSION =~ /^1\.8/
+      @buffer = String.new
+      @io = Snappy.set_encoding io
       write_header!
       if block_given?
         yield self
@@ -34,7 +33,7 @@ module Snappy
       compressed = Snappy.deflate(@buffer)
       @io << [compressed.size, compressed].pack("Na#{compressed.size}")
       @io.flush
-      @buffer = ""
+      @buffer.clear
     end
 
     alias_method :flush, :dump!