extzstd 0.0.3.CONCEPT → 0.3.1

data/ext/libzstd_conf.h

@@ -0,0 +1,8 @@
+#ifndef RUBY_ZSTD_LIBZSTD_CONF_H
+#define RUBY_ZSTD_LIBZSTD_CONF_H 1
+
+#define ZSTD_LEGACY_SUPPORT 1
+#define MEM_MODULE 1
+#define visibility(v) visibility("hidden")
+
+#endif /* RUBY_ZSTD_LIBZSTD_CONF_H */

data/ext/zstd_common.c

@@ -0,0 +1,11 @@
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/common/entropy_common.c"
+#include "../contrib/zstd/lib/common/error_private.c"
+#include "../contrib/zstd/lib/common/fse_decompress.c"
+
+#undef CHECK_F
+#include "../contrib/zstd/lib/common/pool.c"
+#include "../contrib/zstd/lib/common/threading.c"
+#include "../contrib/zstd/lib/common/xxhash.c"
+#include "../contrib/zstd/lib/common/zstd_common.c"

data/ext/zstd_compress.c

@@ -0,0 +1,15 @@
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/compress/fse_compress.c"
+#include "../contrib/zstd/lib/compress/huf_compress.c"
+
+#undef CHECK_F
+#include "../contrib/zstd/lib/compress/zstd_compress.c"
+#include "../contrib/zstd/lib/compress/zstd_compress_literals.c"
+#include "../contrib/zstd/lib/compress/zstd_compress_sequences.c"
+#include "../contrib/zstd/lib/compress/zstd_double_fast.c"
+#include "../contrib/zstd/lib/compress/zstd_fast.c"
+#include "../contrib/zstd/lib/compress/zstd_lazy.c"
+#include "../contrib/zstd/lib/compress/zstd_ldm.c"
+#include "../contrib/zstd/lib/compress/zstd_opt.c"
+#include "../contrib/zstd/lib/compress/hist.c"

data/ext/zstd_decompress.c

@@ -0,0 +1,6 @@
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/decompress/zstd_decompress.c"
+#include "../contrib/zstd/lib/decompress/zstd_decompress_block.c"
+#include "../contrib/zstd/lib/decompress/huf_decompress.c"
+#include "../contrib/zstd/lib/decompress/zstd_ddict.c"

data/ext/zstd_dictbuilder.c

@@ -0,0 +1,10 @@
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/dictBuilder/zdict.c"
+#include "../contrib/zstd/lib/dictBuilder/divsufsort.c"
+
+#undef DISPLAY
+#undef DISPLAYLEVEL
+#undef DISPLAYUPDATE
+#define prime4bytes cover_prime4bytes
+#include "../contrib/zstd/lib/dictBuilder/cover.c"

data/ext/zstd_dictbuilder_fastcover.c

@@ -0,0 +1,3 @@
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/dictBuilder/fastcover.c"

data/ext/zstd_legacy_v01.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v01.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v01.c"

data/ext/zstd_legacy_v02.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v02.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v02.c"

data/ext/zstd_legacy_v03.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v03.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v03.c"

data/ext/zstd_legacy_v04.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v04.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v04.c"

data/ext/zstd_legacy_v05.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v05.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v05.c"

data/ext/zstd_legacy_v06.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v06.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v06.c"

data/ext/zstd_legacy_v07.c

@@ -0,0 +1,3 @@
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v07.c"

data/gemstub.rb

@@ -1,35 +1,41 @@
-
-file "lib/extzstd/version.rb" => %w(README.md) do
-  reg = /^\s*\*\s+version:\s*(\d+(?:\.\w+)+)\s*$/i
-  unless File.read("README.md", mode: "rt") =~ reg
-    raise "``version'' is not defined or bad syntax in ``README.md''"
-  end
-
-  GEMSTUB.version = ver = String($1)
-
-  File.write "lib/extzstd/version.rb", <<-"EOS", mode: "wb"
-module Zstd
-  VERSION = #{ver.inspect}
-end
-  EOS
+verreg = /^\s*\*\s+version:\s*(\d+(?:\.\w+)+)\s*$/i
+unless File.read("README.md", mode: "rt") =~ verreg
+  raise "``version'' is not defined or bad syntax in ``README.md''"
 end
 
-require_relative "lib/extzstd/version"
+version = String($1)
 
 GEMSTUB = Gem::Specification.new do |s|
   s.name = "extzstd"
-  s.version = Zstd::VERSION
+  s.version = version
   s.summary = "ruby bindings for Zstandard (zstd)"
   s.description = <<EOS
-unoficial ruby bindings for Zstandard (zstd) <https://github.com/Cyan4973/zstd>.
+unoficial ruby bindings for Zstandard (zstd) <https://github.com/facebook/zstd>.
 EOS
-  s.homepage = "https://osdn.jp/projects/rutsubo/"
-  s.license = "2-clause BSD License"
+  s.homepage = "https://github.com/dearblue/ruby-extzstd/"
+  s.license = "BSD-2-Clause"
   s.author = "dearblue"
-  s.email = "dearblue@users.osdn.me"
+  s.email = "dearblue@users.noreply.github.com"
 
   s.required_ruby_version = ">= 2.0"
-  s.add_development_dependency "rake"
+  s.add_development_dependency "rake", ">= 12.0"
+end
+
+LIB << "lib/extzstd/version.rb"
+
+file "lib/extzstd/version.rb" => %w(README.md) do
+  GEMSTUB.version = version
+
+  mkpath "lib/extzstd"
+  File.write "lib/extzstd/version.rb", <<-"EOS", mode: "wb"
+module Zstd
+  VERSION = #{version.inspect}
+end
+  EOS
 end
 
 EXTRA.concat(FileList["contrib/**/*"])
+
+filter = %r(contrib/zstd/(?:build|contrib|doc|examples|lib/dll|programs|tests|zlibWrapper))
+DOC.reject! { |path| path =~ filter }
+EXTRA.reject! { |path| path =~ filter }

data/lib/extzstd.rb

@@ -2,7 +2,7 @@
 
 ver = RUBY_VERSION.slice(/\d+\.\d+/)
 soname = File.basename(__FILE__, ".rb") << ".so"
-require_relative File.join(ver, soname)
+require File.join(ver, soname)
 
 require_relative "extzstd/version"
 
@@ -12,44 +12,82 @@ require "stringio"
 # This is ruby bindings for zstd <https://github.com/Cyan4973/zstd> the compression library.
 #
 module Zstd
+  module Internals
+    unless String.method_defined? :b
+      refine String do
+        def b
+          dup.force_encoding(Encoding::BINARY)
+        end
+      end
+    end
+  end
+
+  using Internals
+
+  refine String do
+    def to_zstd(params = nil, dict: nil)
+      ContextLess.encode(self, "".b, nil, dict, params)
+    end
+
+    def unzstd(size = nil, dict: nil)
+      Decoder.open(self, dict) { |d| return d.read(size) }
+    end
+  end
+
+  refine Object do
+    def to_zstd(params = nil, dict: nil, &block)
+      Encoder.open(self, params, dict, &block)
+    end
+
+    def unzstd(dict: nil, &block)
+      Decoder.open(self, dict, &block)
+    end
+  end
+
+  using Zstd
+
   #
   # call-seq:
-  #   encode(src_string, level = nil, dict = nil) -> zstd string
-  #   encode(src_string, encoder_params, dict = nil) -> zstd string
-  #   encode(outport, level = nil, dict = nil) -> zstd encoder
-  #   encode(outport, level = nil, dict = nil) { |encoder| ... } -> yield returned value
-  #   encode(outport, encoder_params, dict = nil) -> zstd encoder
-  #   encode(outport, encoder_params, dict = nil) { |encoder| ... } -> yield returned value
+  #   encode(src_string, level = nil, opts = {}) -> zstd string
+  #   encode(src_string, encode_params, opts = {}) -> zstd string
+  #   encode(outport, level = nil, opts = {}) -> zstd encoder
+  #   encode(outport, level = nil, opts = {}) { |encoder| ... } -> yield returned value
+  #   encode(outport, encode_params, opts = {}) -> zstd encoder
+  #   encode(outport, encode_params, opts = {}) { |encoder| ... } -> yield returned value
   #
-  def self.encode(src, *args, &block)
-    if src.kind_of?(String)
-      dest = Aux::EMPTY_BUFFER.dup
-      return Encoder.open(dest, *args) { |e| e.write src; dest }
-    end
-
-    Encoder.open(src, *args, &block)
+  # [src_string (string)]
+  # [outport (io liked object)]
+  # [level = nil (integer or nil)]
+  # [encode_params (instance of Zstd::Parameters)]
+  # [opts dict: nil (string or nil)]
+  def self.encode(src, *args, **opts, &block)
+    src.to_zstd(*args, **opts, &block)
   end
 
   #
   # call-seq:
-  #   decode(zstd_string, maxsize = nil, dict = nil) -> string
-  #   decode(zstd_stream, maxsize = nil, dict = nil) -> zstd decoder
-  #   decode(zstd_stream, maxsize = nil, dict = nil) { |decoder| ... } -> yield returned value
+  #   decode(zstd_string, maxsize = nil, dict: nil) -> string
+  #   decode(zstd_stream, dict: nil) -> zstd decoder
+  #   decode(zstd_stream, dict: nil) { |decoder| ... } -> yield returned value
   #
-  def self.decode(src, maxsize = nil, dict = nil, &block)
-    if src.kind_of?(String)
-      maxsize &&= maxsize.to_i
-      return Decoder.open(src, dict) { |d| return d.read(maxsize) }
-    end
+  def self.decode(src, *args, **opts, &block)
+    src.unzstd(*args, **opts, &block)
+  end
 
-    Decoder.open(src, dict, &block)
+  class << Zstd
+    alias compress encode
+    alias decompress decode
+    alias uncompress decode
   end
 
-  class Encoder < Struct.new(:encoder, :outport, :destbuf, :status)
+  Compressor = Encoder
+  StreamEncoder = Encoder
+
+  class Encoder
     #
     # call-seq:
     #   open(outport, level = nil, dict = nil) -> zstd encoder
-    #   open(outport, encoder_params, dict = nil) { |encoder| ... } -> yield returned value
+    #   open(outport, encode_params, dict = nil) { |encoder| ... } -> yield returned value
     #
     def self.open(outport, *args)
       e = new(outport, *args)
@@ -59,68 +97,24 @@ module Zstd
       begin
         yield e
       ensure
-        e.close rescue nil unless e.eof?
+        e.close unless e.eof?
       end
     end
 
-    #
-    # call-seq:
-    #   initialize(outport, level = nil, dict = nil)
-    #   initialize(outport, encoder_params, dict = nil)
-    #
-    # +outport+ need has +.<<+ method.
-    #
-    def initialize(outport, params = nil, dict = nil)
-      encoder = BufferedEncoder.new(params, dict)
-      super encoder, outport, "".force_encoding(Encoding::BINARY), [true]
-    end
-
-    def eof
-      !status[0]
-    end
-
-    alias eof? eof
-
-    def close
-      return nil if eof?
-      encoder.end(destbuf, BufferedEncoder.recommended_outsize)
-      outport << destbuf
-      status[0] = false
-      nil
-    end
-
-    def write(buf)
-      raise IOError, "closed stream" if eof?
-
-      off = 0
-      rest = buf.bytesize
-      outsize = BufferedEncoder.recommended_outsize
-      while off && off < rest
-        off = encoder.continue(buf, off, destbuf, outsize)
-        outport << destbuf
-      end
-
-      self
+    def self.encode(src, params = nil, dest: nil, dict: nil)
+      ContextLess.encode(src, dest || "".b, nil, dict, params)
     end
 
-    alias << write
-
-    def flush
-      raise IOError, "closed stream" if eof?
-
-      off = 0
-      encoder.flush(destbuf, BufferedEncoder.recommended_outsize)
-      outport << destbuf
-
-      self
+    class << Encoder
+      alias compress encode
     end
   end
 
-  class Decoder < Struct.new(:decoder, :inport, :readbuf, :destbuf, :status)
-    STATUS_CLOSED = nil
-    STATUS_READY = 0
-    STATUS_INPORT_EOF = 1
+  Uncompressor = Decoder
+  Decompressor = Decoder
+  StreamDecoder = Decoder
 
+  class Decoder
     #
     # call-seq:
     #   open(inport, dict = nil) -> decoder
@@ -143,77 +137,20 @@ module Zstd
       end
     end
 
-    def initialize(inport, dict = nil)
-      raise Error, "require .read method - <%s:0x%08x>" % [inport.class, inport.object_id << 1] unless inport.respond_to?(:read)
-      super(BufferedDecoder.new(dict), inport, StringIO.new(Aux::EMPTY_BUFFER.dup), StringIO.new(Aux::EMPTY_BUFFER.dup), STATUS_READY)
-    end
-
-    def close
-      decoder.reset
-      inport.close rescue nil if inport.respond_to?(:close)
-      readbuf.clear
-      #destbuf.clear
-      self.status = STATUS_CLOSED
-      nil
-    end
-
-    def eof
-      !status
-    end
-
-    alias eof? eof
-
-    def read(size = nil, dest = Aux::EMPTY_BUFFER.dup)
-      dest ||= Aux::EMPTY_BUFFER.dup
-      Aux.change_binary(dest) do
-        #dest.clear
-        dest[0 .. -1] = Aux::EMPTY_BUFFER # keep allocated heap
-
-        return dest if size == 0
-
-        d = Aux::EMPTY_BUFFER.dup
-        until size && size <= 0
-          if destbuf.eof?
-            break unless fetch
-          end
+    def self.decode(src, dest: nil, dict: nil)
+      # NOTE: ContextLess.decode は伸長時のサイズが必要なため、常に利用できるわけではない
+      # ContextLess.decode(src, dest || "".b, nil, dict)
 
-          destbuf.read(size, d)
-          dest << d
-
-          size -= d.bytesize if size
-        end
-      end
-
-      if dest.empty?
-        nil
-      else
-        dest
-      end
+      new(StringIO.new(src), dict).read(nil, dest)
     end
 
-    private
-    def fetch
-      return nil if eof? || status == STATUS_INPORT_EOF
-
-      while true
-        if readbuf.eof?
-          readbuf.string[0 .. -1] = Aux::EMPTY_BUFFER
-          readbuf.rewind
-          unless inport.read(BufferedDecoder.recommended_insize, readbuf.string)
-            self.status = STATUS_INPORT_EOF
-            return nil
-          end
-        end
-
-        off = decoder.continue(readbuf.string, readbuf.pos, destbuf.string, BufferedDecoder.recommended_outsize)
-        readbuf.pos = off if off
-        destbuf.rewind
-        return self if destbuf.size > 0
-      end
+    class << Decoder
+      alias decompress decode
+      alias uncompress decode
     end
   end
 
-  class EncodeParameters
+  class Parameters
     def inspect
       "#<#{self.class} windowlog=#{windowlog}, chainlog=#{chainlog}, " \
         "hashlog=#{hashlog}, searchlog=#{searchlog}, " \
@@ -233,26 +170,10 @@ module Zstd
         q.breakable " "
         q.text "searchlength=#{searchlength},"
         q.breakable " "
+        q.text "targetlength=#{targetlength},"
+        q.breakable " "
         q.text "strategy=#{strategy}>"
       end
     end
   end
-
-  module Aux
-    EMPTY_BUFFER = "".force_encoding(Encoding::BINARY).freeze
-
-    def self.io_read(io, size, buf)
-      raise Error, "encounted EOF (read error)" unless io.read(size, buf)
-      raise Error, "read size too small (read error)" unless buf.bytesize == size
-      buf
-    end
-
-    def self.change_binary(str)
-      e = str.encoding
-      str.force_encoding(Encoding::BINARY)
-      yield
-    ensure
-      str.force_encoding e rescue nil if e
-    end
-  end
 end