extzstd 0.0.2.CONCEPT → 0.0.3.CONCEPT

data/ext/zstd_amalgam.c

@@ -0,0 +1,18 @@
+#define ZSTD_LEGACY_SUPPORT 1
+#define MEM_MODULE 1
+
+#include "../contrib/zstd/compress/zbuff_compress.c"
+#include "../contrib/zstd/compress/fse_compress.c"
+#include "../contrib/zstd/compress/zstd_compress.c"
+#include "../contrib/zstd/compress/huf_compress.c"
+#include "../contrib/zstd/common/xxhash.c"
+#include "../contrib/zstd/common/zstd_common.c"
+#define FSE_abs FSE_abs__redef 
+#include "../contrib/zstd/common/entropy_common.c"
+#include "../contrib/zstd/common/fse_decompress.c"
+#include "../contrib/zstd/decompress/zstd_decompress.c"
+#include "../contrib/zstd/decompress/huf_decompress.c"
+#define ZBUFF_limitCopy ZBUFF_limitCopy__redef
+#include "../contrib/zstd/decompress/zbuff_decompress.c"
+#include "../contrib/zstd/dictBuilder/zdict.c"
+#include "../contrib/zstd/dictBuilder/divsufsort.c"

data/ext/zstd_legacy_v01.c

@@ -0,0 +1 @@
+#include "../contrib/zstd/legacy/zstd_v01.c"

data/ext/zstd_legacy_v02.c

@@ -0,0 +1 @@
+#include "../contrib/zstd/legacy/zstd_v02.c"

data/ext/zstd_legacy_v03.c

@@ -0,0 +1 @@
+#include "../contrib/zstd/legacy/zstd_v03.c"

data/ext/zstd_legacy_v04.c

@@ -0,0 +1 @@
+#include "../contrib/zstd/legacy/zstd_v04.c"

data/ext/zstd_legacy_v05.c

@@ -0,0 +1 @@
+#include "../contrib/zstd/legacy/zstd_v05.c"

data/ext/zstd_legacy_v06.c

@@ -0,0 +1 @@
+#include "../contrib/zstd/legacy/zstd_v06.c"

data/gemstub.rb

@@ -1,3 +1,19 @@
+
+file "lib/extzstd/version.rb" => %w(README.md) do
+  reg = /^\s*\*\s+version:\s*(\d+(?:\.\w+)+)\s*$/i
+  unless File.read("README.md", mode: "rt") =~ reg
+    raise "``version'' is not defined or bad syntax in ``README.md''"
+  end
+
+  GEMSTUB.version = ver = String($1)
+
+  File.write "lib/extzstd/version.rb", <<-"EOS", mode: "wb"
+module Zstd
+  VERSION = #{ver.inspect}
+end
+  EOS
+end
+
 require_relative "lib/extzstd/version"
 
 GEMSTUB = Gem::Specification.new do |s|
@@ -13,7 +29,7 @@ EOS
   s.email = "dearblue@users.osdn.me"
 
   s.required_ruby_version = ">= 2.0"
-  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency "rake"
 end
 
 EXTRA.concat(FileList["contrib/**/*"])

data/lib/extzstd.rb

@@ -1,55 +1,138 @@
-#vim: set fileencoding:utf-8
+#!ruby
 
-ver = RbConfig::CONFIG["ruby_version"].slice(/\d+\.\d+/)
+ver = RUBY_VERSION.slice(/\d+\.\d+/)
 soname = File.basename(__FILE__, ".rb") << ".so"
-lib = File.join(File.dirname(__FILE__), ver, soname)
-if File.file?(lib)
-  require_relative File.join(ver, soname)
-else
-  require_relative soname
-end
+require_relative File.join(ver, soname)
 
 require_relative "extzstd/version"
 
 require "stringio"
 
-if false
-def p(*args)
-  sf = File.basename(caller(1, 1)[0])
-  args.each do |mesg|
-    $stderr.puts "#{sf}: #{mesg.inspect}\n"
+#
+# This is ruby bindings for zstd <https://github.com/Cyan4973/zstd> the compression library.
+#
+module Zstd
+  #
+  # call-seq:
+  #   encode(src_string, level = nil, dict = nil) -> zstd string
+  #   encode(src_string, encoder_params, dict = nil) -> zstd string
+  #   encode(outport, level = nil, dict = nil) -> zstd encoder
+  #   encode(outport, level = nil, dict = nil) { |encoder| ... } -> yield returned value
+  #   encode(outport, encoder_params, dict = nil) -> zstd encoder
+  #   encode(outport, encoder_params, dict = nil) { |encoder| ... } -> yield returned value
+  #
+  def self.encode(src, *args, &block)
+    if src.kind_of?(String)
+      dest = Aux::EMPTY_BUFFER.dup
+      return Encoder.open(dest, *args) { |e| e.write src; dest }
+    end
+
+    Encoder.open(src, *args, &block)
   end
-  return *args
-end
-end
 
-module Zstd
-  module Aux
-    module_function
-    def io_read(io, size, buf)
-      raise Error, "encounted EOF (read error)" unless io.read(size, buf)
-      raise Error, "read size too small (read error)" unless buf.bytesize == size
-      buf
+  #
+  # call-seq:
+  #   decode(zstd_string, maxsize = nil, dict = nil) -> string
+  #   decode(zstd_stream, maxsize = nil, dict = nil) -> zstd decoder
+  #   decode(zstd_stream, maxsize = nil, dict = nil) { |decoder| ... } -> yield returned value
+  #
+  def self.decode(src, maxsize = nil, dict = nil, &block)
+    if src.kind_of?(String)
+      maxsize &&= maxsize.to_i
+      return Decoder.open(src, dict) { |d| return d.read(maxsize) }
     end
+
+    Decoder.open(src, dict, &block)
   end
 
-  class Decoder < Struct.new(:decoder, :import, :readbuf, :destbuf, :status)
-    BLOCKSIZE = 1 << 18
-    STATUS_TERMINATE = nil
-    STATUS_BLOCK_APPROACH = 1
-    STATUS_INBLOCK = 2
+  class Encoder < Struct.new(:encoder, :outport, :destbuf, :status)
+    #
+    # call-seq:
+    #   open(outport, level = nil, dict = nil) -> zstd encoder
+    #   open(outport, encoder_params, dict = nil) { |encoder| ... } -> yield returned value
+    #
+    def self.open(outport, *args)
+      e = new(outport, *args)
+
+      return e unless block_given?
+
+      begin
+        yield e
+      ensure
+        e.close rescue nil unless e.eof?
+      end
+    end
 
     #
     # call-seq:
-    #   open(import) -> decoder
-    #   open(import) { |decoder| ... } -> yield returned value
+    #   initialize(outport, level = nil, dict = nil)
+    #   initialize(outport, encoder_params, dict = nil)
+    #
+    # +outport+ need has +.<<+ method.
     #
-    # [import]
+    def initialize(outport, params = nil, dict = nil)
+      encoder = BufferedEncoder.new(params, dict)
+      super encoder, outport, "".force_encoding(Encoding::BINARY), [true]
+    end
+
+    def eof
+      !status[0]
+    end
+
+    alias eof? eof
+
+    def close
+      return nil if eof?
+      encoder.end(destbuf, BufferedEncoder.recommended_outsize)
+      outport << destbuf
+      status[0] = false
+      nil
+    end
+
+    def write(buf)
+      raise IOError, "closed stream" if eof?
+
+      off = 0
+      rest = buf.bytesize
+      outsize = BufferedEncoder.recommended_outsize
+      while off && off < rest
+        off = encoder.continue(buf, off, destbuf, outsize)
+        outport << destbuf
+      end
+
+      self
+    end
+
+    alias << write
+
+    def flush
+      raise IOError, "closed stream" if eof?
+
+      off = 0
+      encoder.flush(destbuf, BufferedEncoder.recommended_outsize)
+      outport << destbuf
+
+      self
+    end
+  end
+
+  class Decoder < Struct.new(:decoder, :inport, :readbuf, :destbuf, :status)
+    STATUS_CLOSED = nil
+    STATUS_READY = 0
+    STATUS_INPORT_EOF = 1
+
+    #
+    # call-seq:
+    #   open(inport, dict = nil) -> decoder
+    #   open(inport, dict = nil) { |decoder| ... } -> yield returned value
+    #
+    # [inport]
     #   String instance or +read+ method haved Object.
     #
-    def self.open(import)
-      import = StringIO.new(import) if import.kind_of?(String)
-      dec = new(import)
+    def self.open(inport, dict = nil)
+      inport = StringIO.new(inport) if inport.kind_of?(String)
+
+      dec = new(inport, dict)
 
       return dec unless block_given?
 
@@ -60,21 +143,17 @@ module Zstd
       end
     end
 
-    def initialize(import)
-      raise Error, "require .read method - <%s:0x%08x>" % [import.class, import.object_id << 1] unless import.respond_to?(:read)
-      super(LowLevelDecoder.new, import, "".b, "".b, STATUS_BLOCK_APPROACH)
-
-      # read header
-      Aux.io_read(import, decoder.next_srcsize, readbuf)
-      decoder.decode(readbuf, "", 0)
+    def initialize(inport, dict = nil)
+      raise Error, "require .read method - <%s:0x%08x>" % [inport.class, inport.object_id << 1] unless inport.respond_to?(:read)
+      super(BufferedDecoder.new(dict), inport, StringIO.new(Aux::EMPTY_BUFFER.dup), StringIO.new(Aux::EMPTY_BUFFER.dup), STATUS_READY)
     end
 
     def close
       decoder.reset
-      import.close rescue nil if import.respond_to?(:close)
+      inport.close rescue nil if inport.respond_to?(:close)
       readbuf.clear
-      destbuf.clear
-      self.status = STATUS_TERMINATE
+      #destbuf.clear
+      self.status = STATUS_CLOSED
       nil
     end
 
@@ -84,55 +163,96 @@ module Zstd
 
     alias eof? eof
 
-    def read(size = nil, dest = "".b)
-      dest.clear
-      destenc = dest.encoding
-      dest.force_encoding Encoding::BINARY
+    def read(size = nil, dest = Aux::EMPTY_BUFFER.dup)
+      dest ||= Aux::EMPTY_BUFFER.dup
+      Aux.change_binary(dest) do
+        #dest.clear
+        dest[0 .. -1] = Aux::EMPTY_BUFFER # keep allocated heap
 
-      until size && size <= 0
-        if destbuf.empty?
-          unless fetch
-            return nil if dest.empty?
-            break
+        return dest if size == 0
+
+        d = Aux::EMPTY_BUFFER.dup
+        until size && size <= 0
+          if destbuf.eof?
+            break unless fetch
           end
-        end
 
-        d = destbuf.slice!(0, size || destbuf.bytesize)
-        dest << d
+          destbuf.read(size, d)
+          dest << d
 
-        size -= d.bytesize if size
+          size -= d.bytesize if size
+        end
       end
 
-      dest
-    ensure
-      dest.force_encoding destenc rescue nil if destenc
+      if dest.empty?
+        nil
+      else
+        dest
+      end
     end
 
     private
     def fetch
-      return nil if eof?
+      return nil if eof? || status == STATUS_INPORT_EOF
 
       while true
-        if status == STATUS_INBLOCK
-          s = decoder.next_srcsize
-          if s > 0
-            Aux.io_read(import, s, readbuf)
-            return destbuf if decoder.decode(readbuf, destbuf, BLOCKSIZE)
-            next
+        if readbuf.eof?
+          readbuf.string[0 .. -1] = Aux::EMPTY_BUFFER
+          readbuf.rewind
+          unless inport.read(BufferedDecoder.recommended_insize, readbuf.string)
+            self.status = STATUS_INPORT_EOF
+            return nil
           end
-          self.status = STATUS_BLOCK_APPROACH
         end
 
-        # status == STATUS_BLOCK_APPROACH
-        s = decoder.next_srcsize
-        if s == 0
-          self.status = STATUS_TERMINATE
-          return nil
-        end
-        Aux.io_read(import, s, readbuf)
-        decoder.decode(readbuf, "", 0)
-        self.status = STATUS_INBLOCK
+        off = decoder.continue(readbuf.string, readbuf.pos, destbuf.string, BufferedDecoder.recommended_outsize)
+        readbuf.pos = off if off
+        destbuf.rewind
+        return self if destbuf.size > 0
       end
     end
   end
+
+  class EncodeParameters
+    def inspect
+      "#<#{self.class} windowlog=#{windowlog}, chainlog=#{chainlog}, " \
+        "hashlog=#{hashlog}, searchlog=#{searchlog}, " \
+        "searchlength=#{searchlength}, strategy=#{strategy}>"
+    end
+
+    def pretty_print(q)
+      q.group(2, "#<#{self.class}") do
+        q.breakable " "
+        q.text "windowlog=#{windowlog},"
+        q.breakable " "
+        q.text "chainlog=#{chainlog},"
+        q.breakable " "
+        q.text "hashlog=#{hashlog},"
+        q.breakable " "
+        q.text "searchlog=#{searchlog},"
+        q.breakable " "
+        q.text "searchlength=#{searchlength},"
+        q.breakable " "
+        q.text "strategy=#{strategy}>"
+      end
+    end
+  end
+
+  module Aux
+    EMPTY_BUFFER = "".force_encoding(Encoding::BINARY).freeze
+
+    def self.io_read(io, size, buf)
+      raise Error, "encounted EOF (read error)" unless io.read(size, buf)
+      raise Error, "read size too small (read error)" unless buf.bytesize == size
+      buf
+    end
+
+    def self.change_binary(str)
+      e = str.encoding
+      str.force_encoding(Encoding::BINARY)
+      yield
+    ensure
+      str.force_encoding e rescue nil if e
+    end
+  end
 end

data/lib/extzstd/version.rb

@@ -1,3 +1,3 @@
 module Zstd
-  VERSION = "0.0.2.CONCEPT"
+  VERSION = "0.0.3.CONCEPT"
 end

data/test/test_basic.rb

@@ -0,0 +1,41 @@
+#!ruby
+
+require "test-unit"
+require "extzstd"
+require "digest"
+
+class TestZstd < Test::Unit::TestCase
+  def test_encode_decode
+    src = "ABCDEFGabcdefg" * 50
+    assert_equal(src, Zstd.decode(Zstd.encode(src), src.bytesize))
+    #assert_raise(Zstd::Error) { Zstd.decode("", 1111) }
+    d1 = Zstd.encode(src)
+    assert_same(src.tainted?, Zstd.encode(src).tainted?)
+    src1 = src.dup
+    src1.taint
+    assert_same(src1.tainted?, Zstd.encode(src1).tainted?)
+  end
+
+  def test_huge
+    src = "ABCDEFGabcdefg" * 10000000
+    assert_equal(Digest::MD5.hexdigest(src), Digest::MD5.hexdigest(Zstd.decode(Zstd.encode(src))))
+  end
+
+  def test_huge_stream
+    src = "abcdefghijklmnopqrstuvwxyz" * 1000
+    md5a = Digest::MD5.new
+    d = StringIO.new("")
+    Zstd.encode(d) { |z| 1000.times { z << src; md5a.update src } }
+    d.pos = 0
+    md5b = Digest::MD5.new
+    Zstd.decode(d) { |z| buf = ""; while z.read(654321, buf); md5b.update buf; end }
+    assert_equal(md5a.hexdigest, md5b.hexdigest)
+  end
+
+  def test_dictionary
+    dictsrc = "0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz" * 10
+    dict = Zstd.dict_train_from_buffer(dictsrc, 10000)
+    src = "ABCDEFGabcdefg" * 50
+    assert_equal(src, Zstd.decode(Zstd.encode(src, nil, dict), src.bytesize, dict))
+  end
+end

metadata

@@ -1,31 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: extzstd
 version: !ruby/object:Gem::Version
-  version: 0.0.2.CONCEPT
+  version: 0.0.3.CONCEPT
 platform: ruby
 authors:
 - dearblue
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-16 00:00:00.000000000 Z
+date: 2016-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
-description: |
-  unoficial ruby bindings for Zstandard (zstd) <https://github.com/Cyan4973/zstd>.
+        version: '0'
+description: 'unoficial ruby bindings for Zstandard (zstd) <https://github.com/Cyan4973/zstd>.
+
+'
 email: dearblue@users.osdn.me
 executables: []
 extensions:
@@ -34,9 +35,18 @@ extra_rdoc_files:
 - LICENSE
 - README.md
 - contrib/zstd/LICENSE
-- ext/extzstd-stream.c
+- contrib/zstd/README.md
 - ext/extzstd.c
 - ext/extzstd.h
+- ext/extzstd_buffered.c
+- ext/extzstd_nogvls.h
+- ext/zstd_amalgam.c
+- ext/zstd_legacy_v01.c
+- ext/zstd_legacy_v02.c
+- ext/zstd_legacy_v03.c
+- ext/zstd_legacy_v04.c
+- ext/zstd_legacy_v05.c
+- ext/zstd_legacy_v06.c
 - lib/extzstd.rb
 - lib/extzstd/version.rb
 files:
@@ -44,21 +54,62 @@ files:
 - README.md
 - Rakefile
 - contrib/zstd/LICENSE
-- contrib/zstd/Makefile
-- contrib/zstd/fse.c
-- contrib/zstd/fse.h
-- contrib/zstd/fse_static.h
-- contrib/zstd/libzstd.pc.in
-- contrib/zstd/zstd.c
-- contrib/zstd/zstd.h
-- contrib/zstd/zstd_static.h
+- contrib/zstd/README.md
+- contrib/zstd/common/bitstream.h
+- contrib/zstd/common/entropy_common.c
+- contrib/zstd/common/error_private.h
+- contrib/zstd/common/error_public.h
+- contrib/zstd/common/fse.h
+- contrib/zstd/common/fse_decompress.c
+- contrib/zstd/common/huf.h
+- contrib/zstd/common/mem.h
+- contrib/zstd/common/xxhash.c
+- contrib/zstd/common/xxhash.h
+- contrib/zstd/common/zbuff.h
+- contrib/zstd/common/zstd.h
+- contrib/zstd/common/zstd_common.c
+- contrib/zstd/common/zstd_internal.h
+- contrib/zstd/compress/fse_compress.c
+- contrib/zstd/compress/huf_compress.c
+- contrib/zstd/compress/zbuff_compress.c
+- contrib/zstd/compress/zstd_compress.c
+- contrib/zstd/compress/zstd_opt.h
+- contrib/zstd/decompress/huf_decompress.c
+- contrib/zstd/decompress/zbuff_decompress.c
+- contrib/zstd/decompress/zstd_decompress.c
+- contrib/zstd/dictBuilder/divsufsort.c
+- contrib/zstd/dictBuilder/divsufsort.h
+- contrib/zstd/dictBuilder/zdict.c
+- contrib/zstd/dictBuilder/zdict.h
+- contrib/zstd/legacy/zstd_legacy.h
+- contrib/zstd/legacy/zstd_v01.c
+- contrib/zstd/legacy/zstd_v01.h
+- contrib/zstd/legacy/zstd_v02.c
+- contrib/zstd/legacy/zstd_v02.h
+- contrib/zstd/legacy/zstd_v03.c
+- contrib/zstd/legacy/zstd_v03.h
+- contrib/zstd/legacy/zstd_v04.c
+- contrib/zstd/legacy/zstd_v04.h
+- contrib/zstd/legacy/zstd_v05.c
+- contrib/zstd/legacy/zstd_v05.h
+- contrib/zstd/legacy/zstd_v06.c
+- contrib/zstd/legacy/zstd_v06.h
 - ext/extconf.rb
-- ext/extzstd-stream.c
 - ext/extzstd.c
 - ext/extzstd.h
+- ext/extzstd_buffered.c
+- ext/extzstd_nogvls.h
+- ext/zstd_amalgam.c
+- ext/zstd_legacy_v01.c
+- ext/zstd_legacy_v02.c
+- ext/zstd_legacy_v03.c
+- ext/zstd_legacy_v04.c
+- ext/zstd_legacy_v05.c
+- ext/zstd_legacy_v06.c
 - gemstub.rb
 - lib/extzstd.rb
 - lib/extzstd/version.rb
+- test/test_basic.rb
 homepage: https://osdn.jp/projects/rutsubo/
 licenses:
 - 2-clause BSD License
@@ -83,9 +134,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: ruby bindings for Zstandard (zstd)
 test_files: []
-has_rdoc: