snappy 0.0.17 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 69133bb0d8adece5bc23fcf0db6e66180c58b04c
-  data.tar.gz: 410ae9d85c9260000a0b816efe4954a92f4188d9
+SHA256:
+  metadata.gz: 3b633f54dc239a8a5a4ef4df91fd0a23a100298ca83192d30096490f6174670b
+  data.tar.gz: 7caed929e615860f65dc0a25568cbe1c4f01ed377fbbc96de457acffb9dcc349
 SHA512:
-  metadata.gz: 18927ee51e2ddca5bc23c1fa80d207f8b9c2ae9450085606153233dd000c1c1f72bf14f43a2248d1290328bfde776760e68800d10526447b96cd975824b7528c
-  data.tar.gz: 8f9ed859c2c472a408425d1981c63a92c0c7349d1e1a4426b422dd087d89f70167ed0c80e009f581c0f348d65afa5ef017cb2ee96f3620049f921f0260c09f36
+  metadata.gz: 6c51f97d066a0bc256498157c9e9a2dab2da1191f310da24d0b54531de6ceb4db09471a4d8738180e20ca06f8e7147440ded43c6ad6baf1599f626c751a9086a
+  data.tar.gz: 8d8b5494df7e5858ebe77c2a86845257409cd71d793f8cd7c2123f9f784e75fb0d66294ce87ddd5fc5140b4702a135e8a644b5a07308414a9fc747fb6111658a

data/.travis.yml

@@ -9,12 +9,17 @@ rvm:
   - 2.2
   - 2.3
   - 2.4
+  - 2.5
   - jruby-19mode
 env:
   global:
     - PATH="/usr/lib/ccache:$PATH"
 before_install:
-  - gem install bundler
+  - travis_retry gem install bundler --no-document -v 1.16.1
+addons:
+  apt:
+    packages:
+      - cmake
 deploy:
   provider: rubygems
   api_key:

data/Gemfile

@@ -2,3 +2,8 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in snappy.gemspec
 gemspec
+
+group :development do
+  gem 'rake'
+  gem 'minitest'
+end

data/README.md

@@ -4,13 +4,27 @@ see https://github.com/google/snappy
 
 ## Preparation
 
-Use libsnappy
 
-    $ brew install snappy
+### macOS
 
-Or
+```bash
+$ brew install snappy
+$ brew install autoconf automake libtool
+```
+
+### Ubuntu
+
+```bash
+$ apt-get install libsnappy-dev -y
+$ apt-get install libtool automake autoconf -y
+```
+
+### Alpine
 
-    $ brew install autoconf automake libtool
+```bash
+$ apk install snappy
+$ apk install build-base libexecinfo automake autoconf libtool
+```
 
 ## Installation
 
@@ -28,6 +42,7 @@ Or install it yourself as:
 
 ## Usage
 
+Snappy-java format
 ```ruby
 require 'snappy'
 
@@ -38,6 +53,15 @@ Snappy.inflate(source)
 # => Decompressed data
 ```
 
+Hadoop-snappy format
+```ruby
+Snappy::Hadoop.deflate(source)
+# => Compressed data
+
+Snappy::Hadoop.inflate(source)
+# => Decompressed data
+```
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -5,6 +5,7 @@ require "rbconfig"
 DLEXT = RbConfig::CONFIG['DLEXT']
 
 Rake::TestTask.new do |t|
+  t.pattern = 'test/**/test-*.rb'
   t.warning = true
   t.verbose = true
 end

data/ext/extconf.rb

@@ -6,24 +6,29 @@ unless pkg_config('libsnappy') || have_library('snappy')
   pwd = File.dirname File.expand_path __FILE__
   dir = File.join pwd, '..', 'vendor', 'snappy'
 
-  Dir.chdir dir do
-    raise '`autogen.sh` failed' unless system './autogen.sh'
-    raise '`configure` failed'  unless system './configure --disable-option-checking --disable-dependency-tracking --disable-gtest --without-gflags'
+  Dir.chdir(dir) do
+    FileUtils.mkdir_p 'build'
+    Dir.chdir(File.join(dir, 'build')) do
+      `cmake .. -DCMAKE_BUILD_TYPE=Release`
+    end
   end
 
-  src = %w(
-   config.h
-   snappy-c.cc
-   snappy-c.h
-   snappy-internal.h
-   snappy-sinksource.cc
-   snappy-sinksource.h
-   snappy-stubs-internal.cc
-   snappy-stubs-internal.h
-   snappy-stubs-public.h
-   snappy.cc
-   snappy.h
-  ).map { |e| File.join dir, e }
+  src = %w[
+    config.h
+    snappy-stubs-public.h
+  ].map { |e| File.join dir, 'build', e }
+  FileUtils.cp src, pwd, :verbose => true
+  src = %w[
+    snappy-c.cc
+    snappy-c.h
+    snappy-internal.h
+    snappy-sinksource.cc
+    snappy-sinksource.h
+    snappy.cc
+    snappy.h
+    snappy-stubs-internal.cc
+    snappy-stubs-internal.h
+  ].map { |e| File.join dir, e }
   FileUtils.cp src, pwd, :verbose => true
 end
 

data/lib/snappy.rb

@@ -5,3 +5,4 @@ require 'snappy-jars' if defined?(JRUBY_VERSION)
 require 'snappy_ext'
 require 'snappy/reader'
 require 'snappy/writer'
+require 'snappy/hadoop'

data/lib/snappy/hadoop.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/hadoop/reader'
+require 'snappy/hadoop/writer'
+
+module Snappy
+  module Hadoop
+    def self.deflate(source, block_size = Snappy::Hadoop::Writer::DEFAULT_BLOCK_SIZE)
+      compressed_io = StringIO.new
+      writer = Snappy::Hadoop::Writer.new(compressed_io)
+      writer << source
+      writer.flush
+      compressed_io.string
+    end
+
+    def self.inflate(source)
+      Snappy::Hadoop::Reader.new(StringIO.new(source)).read
+    end
+  end
+end

data/lib/snappy/hadoop/reader.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Reader
+      attr_reader :io
+
+      def initialize(io)
+        @io = Snappy.set_encoding io
+        yield self if block_given?
+      end
+
+      def each
+        until @io.eof?
+          # Uncompressed size (32 bit integer, BE).
+          uncompressed_size = @io.read(4).unpack('N').first
+
+          uncompressed_block_io = StringIO.new
+          while uncompressed_block_io.size < uncompressed_size
+            # Compressed subblock size (32 bit integer, BE).
+            compressed_size = @io.read(4).unpack('N').first
+            uncompressed_block_io << Snappy.inflate(@io.read(compressed_size))
+          end
+
+          if uncompressed_block_io.size > uncompressed_size
+            raise RuntimeError("Invalid data: expected #{uncompressed_size} bytes, got #{Uncompressed.size}")
+          end
+
+          yield uncompressed_block_io.string if block_given?
+        end
+      end
+
+      def read
+        buff = StringIO.new
+        each do |chunk|
+          buff << chunk
+        end
+        buff.string
+      end
+
+      def each_line(sep_string = $/)
+        last = ""
+        each do |chunk|
+          chunk = last + chunk
+          lines = chunk.split(sep_string)
+          last = lines.pop
+          lines.each do |line|
+            yield line if block_given?
+          end
+        end
+        yield last
+      end
+    end
+  end
+end

data/lib/snappy/hadoop/writer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'snappy/shim'
+
+module Snappy
+  module Hadoop
+    class Writer
+      DEFAULT_BLOCK_SIZE = 256 * 1024
+
+      attr_reader :io, :block_size
+
+      def initialize(io, block_size = DEFAULT_BLOCK_SIZE)
+        @block_size = block_size
+        @buffer = String.new
+        @io = Snappy.set_encoding io
+        if block_given?
+          yield self
+          dump!
+        end
+      end
+
+      def <<(msg)
+        @buffer << msg.to_s
+        dump! if @buffer.size >= @block_size
+      end
+
+      alias_method :write, :<<
+
+      def dump!
+        offset = 0
+        while offset < @buffer.size
+          uncompressed = @buffer[offset, @block_size]
+          compressed = Snappy.deflate(uncompressed)
+
+          # Uncompressed size (32 bit integer, BE), compressed size (32 bit integer, BE), data.
+          @io << [uncompressed.size, compressed.size, compressed].pack("NNa#{compressed.size}")
+          offset += uncompressed.size
+        end
+
+        @io.flush
+        @buffer.clear
+      end
+
+      alias_method :flush, :dump!
+
+      def close
+        @io.close
+      end
+    end
+  end
+end

data/lib/snappy/reader.rb

@@ -19,17 +19,17 @@ module Snappy
           size = @io.read(4).unpack('N').first
           yield Snappy.inflate(@io.read(size)) if block_given?
         else
-          yield Snappy.inflate @io.read if block_given?
+          yield Snappy.inflate(@io.read) if block_given?
         end
       end
     end
 
     def read
-      @buff = StringIO.new
+      buff = StringIO.new
       each do |chunk|
-        @buff << chunk
+        buff << chunk
       end
-      @buff.string
+      buff.string
     end
 
     def each_line(sep_string = $/)

data/lib/snappy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Snappy
-  VERSION = "0.0.17"
+  VERSION = "0.1.0"
 end

data/smoke.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 gem list | grep snappy && gem uninstall --force snappy
 bundle exec rake clean build
-gem install --no-ri --no-rdoc "$(ls pkg/snappy-*.gem)"
+gem install --no-document "$(ls pkg/snappy-*.gem)"
 cat <<EOF | ruby
 require 'snappy'
 abort if Snappy.inflate(Snappy.deflate(File.read('smoke.sh'))) != File.read('smoke.sh')

data/snappy.gemspec

@@ -29,8 +29,4 @@ Gem::Specification.new do |spec|
   else
     spec.extensions = ["ext/extconf.rb"]
   end
-
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "minitest"
 end

data/test/hadoop/test-snappy-hadoop-reader.rb

@@ -0,0 +1,103 @@
+require "minitest/autorun"
+require "minitest/spec"
+require "snappy"
+require "stringio"
+
+describe Snappy::Hadoop::Reader do
+  before do
+    @buffer = StringIO.new
+    Snappy::Hadoop::Writer.new @buffer do |w|
+      w << "foo"
+      w << "bar"
+      w << "baz"
+      w << "quux"
+    end
+    @buffer.rewind
+  end
+
+  subject do
+    Snappy::Hadoop::Reader.new @buffer
+  end
+
+  describe :initialize do
+    it "should yield itself to the block" do
+      yielded = nil
+      returned = Snappy::Hadoop::Reader.new @buffer do |r|
+        yielded = r
+      end
+      returned.must_equal yielded
+    end
+  end
+
+  describe :io do
+    it "should be a constructor argument" do
+      subject.io.must_equal @buffer
+    end
+
+    it "should not receive `length' in initializing" do
+      length = MiniTest::Mock.new.expect(:call, 0)
+      @buffer.stub(:length, length) do
+        Snappy::Hadoop::Reader.new @buffer
+      end
+      -> { length.verify }.must_raise MockExpectationError
+    end
+  end
+
+  describe :each do
+    before do
+      Snappy::Hadoop::Writer.new @buffer do |w|
+        w << "foo"
+        w << "bar"
+        w.dump!
+        w << "baz"
+        w << "quux"
+      end
+      @buffer.rewind
+    end
+
+    it "should yield each chunk" do
+      chunks = []
+      Snappy::Hadoop::Reader.new(@buffer).each do |chunk|
+        chunks << chunk
+      end
+      chunks.must_equal ["foobar", "bazquux"]
+    end
+  end
+
+  describe :read do
+    before do
+      Snappy::Hadoop::Writer.new @buffer do |w|
+        w << "foo"
+        w << "bar"
+        w << "baz"
+        w << "quux"
+      end
+      @buffer.rewind
+    end
+
+    it "should return the bytes" do
+      Snappy::Hadoop::Reader.new(@buffer).read.must_equal "foobarbazquux"
+    end
+  end
+
+  describe :each_line do
+    before do
+      Snappy::Hadoop::Writer.new @buffer do |w|
+        w << "foo\n"
+        w << "bar"
+        w.dump!
+        w << "baz\n"
+        w << "quux\n"
+      end
+      @buffer.rewind
+    end
+
+    it "should yield each line" do
+      lines = []
+      Snappy::Hadoop::Reader.new(@buffer).each_line do |line|
+        lines << line
+      end
+      lines.must_equal ["foo", "barbaz", "quux"]
+    end
+  end
+end