extzstd 0.0.3.CONCEPT → 0.1

data/lib/extzstd.rb

@@ -14,32 +14,46 @@ require "stringio"
 module Zstd
   #
   # call-seq:
-  #   encode(src_string, level = nil, dict = nil) -> zstd string
-  #   encode(src_string, encoder_params, dict = nil) -> zstd string
-  #   encode(outport, level = nil, dict = nil) -> zstd encoder
-  #   encode(outport, level = nil, dict = nil) { |encoder| ... } -> yield returned value
-  #   encode(outport, encoder_params, dict = nil) -> zstd encoder
-  #   encode(outport, encoder_params, dict = nil) { |encoder| ... } -> yield returned value
+  #   encode(src_string, level = nil, opts = {}) -> zstd string
+  #   encode(src_string, encode_params, opts = {}) -> zstd string
+  #   encode(outport, level = nil, opts = {}) -> zstd encoder
+  #   encode(outport, level = nil, opts = {}) { |encoder| ... } -> yield returned value
+  #   encode(outport, encode_params, opts = {}) -> zstd encoder
+  #   encode(outport, encode_params, opts = {}) { |encoder| ... } -> yield returned value
   #
-  def self.encode(src, *args, &block)
+  # [src_string (string)]
+  # [outport (io liked object)]
+  # [level = nil (integer or nil)]
+  # [encode_params (instance of Zstd::Parameters)]
+  # [opts dict: nil (string or nil)]
+  def self.encode(src, params = nil, dict: nil, &block)
     if src.kind_of?(String)
-      dest = Aux::EMPTY_BUFFER.dup
-      return Encoder.open(dest, *args) { |e| e.write src; dest }
+      return ContextLess.encode(src, Aux::EMPTY_BUFFER.dup, nil, dict, params)
     end
 
-    Encoder.open(src, *args, &block)
+    Encoder.open(src, params, dict, &block)
   end
 
   #
   # call-seq:
-  #   decode(zstd_string, maxsize = nil, dict = nil) -> string
-  #   decode(zstd_stream, maxsize = nil, dict = nil) -> zstd decoder
-  #   decode(zstd_stream, maxsize = nil, dict = nil) { |decoder| ... } -> yield returned value
+  #   decode(zstd_string, maxsize = nil, dict: nil) -> string
+  #   decode(zstd_stream, dict: nil) -> zstd decoder
+  #   decode(zstd_stream, dict: nil) { |decoder| ... } -> yield returned value
   #
-  def self.decode(src, maxsize = nil, dict = nil, &block)
+  def self.decode(src, *args, dict: nil, &block)
     if src.kind_of?(String)
-      maxsize &&= maxsize.to_i
-      return Decoder.open(src, dict) { |d| return d.read(maxsize) }
+      case args.size
+      when 0
+        return ContextLess.decode(src, Aux::EMPTY_BUFFER.dup, nil, dict)
+      when 1
+        Decoder.open(src, dict) { |d| return d.read(args[0].to_i) }
+      else
+        raise ArgumentError, "wrong argument number (given #{args.size}, expect 1 or 2)"
+      end
+    end
+
+    unless args.empty?
+      raise ArgumentError, "wrong argument number (given #{args.size}, expect 1)"
     end
 
     Decoder.open(src, dict, &block)
@@ -49,7 +63,7 @@ module Zstd
     #
     # call-seq:
     #   open(outport, level = nil, dict = nil) -> zstd encoder
-    #   open(outport, encoder_params, dict = nil) { |encoder| ... } -> yield returned value
+    #   open(outport, encode_params, dict = nil) { |encoder| ... } -> yield returned value
     #
     def self.open(outport, *args)
       e = new(outport, *args)
@@ -66,12 +80,12 @@ module Zstd
     #
     # call-seq:
     #   initialize(outport, level = nil, dict = nil)
-    #   initialize(outport, encoder_params, dict = nil)
+    #   initialize(outport, encode_params, dict = nil)
     #
     # +outport+ need has +.<<+ method.
     #
     def initialize(outport, params = nil, dict = nil)
-      encoder = BufferedEncoder.new(params, dict)
+      encoder = StreamEncoder.new(params, dict)
       super encoder, outport, "".force_encoding(Encoding::BINARY), [true]
     end
 
@@ -83,7 +97,7 @@ module Zstd
 
     def close
       return nil if eof?
-      encoder.end(destbuf, BufferedEncoder.recommended_outsize)
+      encoder.end(destbuf, StreamEncoder::OUTSIZE)
       outport << destbuf
       status[0] = false
       nil
@@ -94,9 +108,9 @@ module Zstd
 
       off = 0
       rest = buf.bytesize
-      outsize = BufferedEncoder.recommended_outsize
+      outsize = StreamEncoder::OUTSIZE
       while off && off < rest
-        off = encoder.continue(buf, off, destbuf, outsize)
+        off = encoder.update(buf, off, destbuf, outsize)
         outport << destbuf
       end
 
@@ -109,14 +123,16 @@ module Zstd
       raise IOError, "closed stream" if eof?
 
       off = 0
-      encoder.flush(destbuf, BufferedEncoder.recommended_outsize)
+      encoder.flush(destbuf, StreamEncoder::OUTSIZE)
       outport << destbuf
 
       self
     end
   end
 
-  class Decoder < Struct.new(:decoder, :inport, :readbuf, :destbuf, :status)
+  class Decoder
+    attr_reader :decoder, :inport, :readbuf, :destbuf, :status, :pos
+
     STATUS_CLOSED = nil
     STATUS_READY = 0
     STATUS_INPORT_EOF = 1
@@ -145,15 +161,19 @@ module Zstd
 
     def initialize(inport, dict = nil)
       raise Error, "require .read method - <%s:0x%08x>" % [inport.class, inport.object_id << 1] unless inport.respond_to?(:read)
-      super(BufferedDecoder.new(dict), inport, StringIO.new(Aux::EMPTY_BUFFER.dup), StringIO.new(Aux::EMPTY_BUFFER.dup), STATUS_READY)
+      @decoder = StreamDecoder.new(dict)
+      @inport = inport
+      @readbuf = StringIO.new(Aux::EMPTY_BUFFER.dup)
+      @destbuf = StringIO.new(Aux::EMPTY_BUFFER.dup)
+      @status = STATUS_READY
+      @pos = 0
     end
 
     def close
-      decoder.reset
       inport.close rescue nil if inport.respond_to?(:close)
-      readbuf.clear
-      #destbuf.clear
-      self.status = STATUS_CLOSED
+      readbuf.truncate 0
+      destbuf.truncate 0
+      @status = STATUS_CLOSED
       nil
     end
 
@@ -165,11 +185,11 @@ module Zstd
 
     def read(size = nil, dest = Aux::EMPTY_BUFFER.dup)
       dest ||= Aux::EMPTY_BUFFER.dup
+      size &&= size.to_i
       Aux.change_binary(dest) do
         #dest.clear
-        dest[0 .. -1] = Aux::EMPTY_BUFFER # keep allocated heap
-
-        return dest if size == 0
+        dest[0 .. -1] = Aux::EMPTY_BUFFER unless dest.empty? # keep allocated heap
+        return dest unless !size || size > 0
 
         d = Aux::EMPTY_BUFFER.dup
         until size && size <= 0
@@ -187,33 +207,45 @@ module Zstd
       if dest.empty?
         nil
       else
+        @pos += dest.bytesize
         dest
       end
     end
 
     private
     def fetch
-      return nil if eof? || status == STATUS_INPORT_EOF
+      return nil if eof?
+
+      destbuf.rewind
+      destbuf.truncate 0
 
       while true
-        if readbuf.eof?
-          readbuf.string[0 .. -1] = Aux::EMPTY_BUFFER
+        if readbuf.eof? && status != STATUS_INPORT_EOF
           readbuf.rewind
-          unless inport.read(BufferedDecoder.recommended_insize, readbuf.string)
-            self.status = STATUS_INPORT_EOF
-            return nil
+          unless inport.read StreamDecoder::INSIZE, readbuf.string
+            @status = STATUS_INPORT_EOF
           end
         end
 
-        off = decoder.continue(readbuf.string, readbuf.pos, destbuf.string, BufferedDecoder.recommended_outsize)
-        readbuf.pos = off if off
-        destbuf.rewind
-        return self if destbuf.size > 0
+        begin
+          off = decoder.update readbuf.string, readbuf.pos, destbuf.string, StreamDecoder::OUTSIZE
+          readbuf.pos = off if off
+          return self if destbuf.size > 0
+          break if readbuf.eof? && status == STATUS_INPORT_EOF
+        rescue Zstd::InitMissingError
+          break if readbuf.eof? && status == STATUS_INPORT_EOF
+          raise
+        end
       end
+
+      # when readbuf.eof? && status == STATUS_INPORT_EOF
+
+      @status = nil
+      nil
     end
   end
 
-  class EncodeParameters
+  class Parameters
     def inspect
       "#<#{self.class} windowlog=#{windowlog}, chainlog=#{chainlog}, " \
         "hashlog=#{hashlog}, searchlog=#{searchlog}, " \
@@ -233,6 +265,8 @@ module Zstd
         q.breakable " "
         q.text "searchlength=#{searchlength},"
         q.breakable " "
+        q.text "targetlength=#{targetlength},"
+        q.breakable " "
         q.text "strategy=#{strategy}>"
       end
     end

data/test/test_basic.rb

@@ -18,24 +18,29 @@ class TestZstd < Test::Unit::TestCase
 
   def test_huge
     src = "ABCDEFGabcdefg" * 10000000
-    assert_equal(Digest::MD5.hexdigest(src), Digest::MD5.hexdigest(Zstd.decode(Zstd.encode(src))))
+    resrc = Zstd.decode(Zstd.encode(src))
+    assert_equal(src.bytesize, resrc.bytesize)
+    assert_equal(Digest::MD5.hexdigest(src), Digest::MD5.hexdigest(resrc))
   end
 
   def test_huge_stream
     src = "abcdefghijklmnopqrstuvwxyz" * 1000
+    size_a = 0
     md5a = Digest::MD5.new
     d = StringIO.new("")
-    Zstd.encode(d) { |z| 1000.times { z << src; md5a.update src } }
+    Zstd.encode(d) { |z| 1000.times { z << src; size_a += src.bytesize; md5a.update src } }
     d.pos = 0
+    size_b = 0
     md5b = Digest::MD5.new
-    Zstd.decode(d) { |z| buf = ""; while z.read(654321, buf); md5b.update buf; end }
+    Zstd.decode(d) { |z| buf = ""; while z.read(654321, buf); size_b += buf.bytesize; md5b.update buf; end }
+    assert_equal size_a, size_b
     assert_equal(md5a.hexdigest, md5b.hexdigest)
   end
 
   def test_dictionary
-    dictsrc = "0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz" * 10
-    dict = Zstd.dict_train_from_buffer(dictsrc, 10000)
+    dictsrc = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_-" * 5
+    dict = Zstd::Dictionary.train_from_buffer(dictsrc, 10000)
     src = "ABCDEFGabcdefg" * 50
-    assert_equal(src, Zstd.decode(Zstd.encode(src, nil, dict), src.bytesize, dict))
+    assert_equal(src, Zstd.decode(Zstd.encode(src, dict: dict), src.bytesize, dict: dict))
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: extzstd
 version: !ruby/object:Gem::Version
-  version: 0.0.3.CONCEPT
+  version: '0.1'
 platform: ruby
 authors:
 - dearblue
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-31 00:00:00.000000000 Z
+date: 2016-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -32,24 +32,30 @@ executables: []
 extensions:
 - ext/extconf.rb
 extra_rdoc_files:
+- HISTORY.ja
 - LICENSE
 - README.md
 - contrib/zstd/LICENSE
 - contrib/zstd/README.md
 - ext/extzstd.c
 - ext/extzstd.h
-- ext/extzstd_buffered.c
 - ext/extzstd_nogvls.h
-- ext/zstd_amalgam.c
+- ext/extzstd_stream.c
+- ext/zstd_common.c
+- ext/zstd_compress.c
+- ext/zstd_decompress.c
+- ext/zstd_dictbuilder.c
 - ext/zstd_legacy_v01.c
 - ext/zstd_legacy_v02.c
 - ext/zstd_legacy_v03.c
 - ext/zstd_legacy_v04.c
 - ext/zstd_legacy_v05.c
 - ext/zstd_legacy_v06.c
+- ext/zstd_legacy_v07.c
 - lib/extzstd.rb
 - lib/extzstd/version.rb
 files:
+- HISTORY.ja
 - LICENSE
 - README.md
 - Rakefile
@@ -57,8 +63,8 @@ files:
 - contrib/zstd/README.md
 - contrib/zstd/common/bitstream.h
 - contrib/zstd/common/entropy_common.c
+- contrib/zstd/common/error_private.c
 - contrib/zstd/common/error_private.h
-- contrib/zstd/common/error_public.h
 - contrib/zstd/common/fse.h
 - contrib/zstd/common/fse_decompress.c
 - contrib/zstd/common/huf.h
@@ -66,8 +72,8 @@ files:
 - contrib/zstd/common/xxhash.c
 - contrib/zstd/common/xxhash.h
 - contrib/zstd/common/zbuff.h
-- contrib/zstd/common/zstd.h
 - contrib/zstd/common/zstd_common.c
+- contrib/zstd/common/zstd_errors.h
 - contrib/zstd/common/zstd_internal.h
 - contrib/zstd/compress/fse_compress.c
 - contrib/zstd/compress/huf_compress.c
@@ -94,18 +100,25 @@ files:
 - contrib/zstd/legacy/zstd_v05.h
 - contrib/zstd/legacy/zstd_v06.c
 - contrib/zstd/legacy/zstd_v06.h
+- contrib/zstd/legacy/zstd_v07.c
+- contrib/zstd/legacy/zstd_v07.h
+- contrib/zstd/zstd.h
 - ext/extconf.rb
 - ext/extzstd.c
 - ext/extzstd.h
-- ext/extzstd_buffered.c
 - ext/extzstd_nogvls.h
-- ext/zstd_amalgam.c
+- ext/extzstd_stream.c
+- ext/zstd_common.c
+- ext/zstd_compress.c
+- ext/zstd_decompress.c
+- ext/zstd_dictbuilder.c
 - ext/zstd_legacy_v01.c
 - ext/zstd_legacy_v02.c
 - ext/zstd_legacy_v03.c
 - ext/zstd_legacy_v04.c
 - ext/zstd_legacy_v05.c
 - ext/zstd_legacy_v06.c
+- ext/zstd_legacy_v07.c
 - gemstub.rb
 - lib/extzstd.rb
 - lib/extzstd/version.rb
@@ -129,9 +142,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.4

data/contrib/zstd/common/error_public.h

@@ -1,77 +0,0 @@
-/* ******************************************************************
-   Error codes list
-   Copyright (C) 2016, Yann Collet
-
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - Homepage : http://www.zstd.net
-****************************************************************** */
-#ifndef ERROR_PUBLIC_H_MODULE
-#define ERROR_PUBLIC_H_MODULE
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* ****************************************
-*  error codes list
-******************************************/
-typedef enum {
-  ZSTD_error_no_error,
-  ZSTD_error_GENERIC,
-  ZSTD_error_prefix_unknown,
-  ZSTD_error_frameParameter_unsupported,
-  ZSTD_error_frameParameter_unsupportedBy32bits,
-  ZSTD_error_compressionParameter_unsupported,
-  ZSTD_error_init_missing,
-  ZSTD_error_memory_allocation,
-  ZSTD_error_stage_wrong,
-  ZSTD_error_dstSize_tooSmall,
-  ZSTD_error_srcSize_wrong,
-  ZSTD_error_corruption_detected,
-  ZSTD_error_checksum_wrong,
-  ZSTD_error_tableLog_tooLarge,
-  ZSTD_error_maxSymbolValue_tooLarge,
-  ZSTD_error_maxSymbolValue_tooSmall,
-  ZSTD_error_dictionary_corrupted,
-  ZSTD_error_dictionary_wrong,
-  ZSTD_error_maxCode
-} ZSTD_ErrorCode;
-
-/*! ZSTD_getErrorCode() :
-    convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
-    which can be used to compare directly with enum list published into "error_public.h" */
-ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif /* ERROR_PUBLIC_H_MODULE */