extzstd 0.1 → 0.3.2

data/ext/libzstd_conf.h

@@ -0,0 +1,8 @@
+#ifndef RUBY_ZSTD_LIBZSTD_CONF_H
+#define RUBY_ZSTD_LIBZSTD_CONF_H 1
+
+#define ZSTD_LEGACY_SUPPORT 1
+#define MEM_MODULE 1
+#define visibility(v) visibility("hidden")
+
+#endif /* RUBY_ZSTD_LIBZSTD_CONF_H */

data/ext/zstd_common.c

@@ -1,8 +1,11 @@
-#define ZSTD_LEGACY_SUPPORT 1
-#define MEM_MODULE 1
+#include "libzstd_conf.h"
 
-#include "../contrib/zstd/common/entropy_common.c"
-#include "../contrib/zstd/common/error_private.c"
-#include "../contrib/zstd/common/fse_decompress.c"
-#include "../contrib/zstd/common/xxhash.c"
-#include "../contrib/zstd/common/zstd_common.c"
+#include "../contrib/zstd/lib/common/entropy_common.c"
+#include "../contrib/zstd/lib/common/error_private.c"
+#include "../contrib/zstd/lib/common/fse_decompress.c"
+
+#undef CHECK_F
+#include "../contrib/zstd/lib/common/pool.c"
+#include "../contrib/zstd/lib/common/threading.c"
+#include "../contrib/zstd/lib/common/xxhash.c"
+#include "../contrib/zstd/lib/common/zstd_common.c"

data/ext/zstd_compress.c

@@ -1,6 +1,15 @@
-#define ZSTD_LEGACY_SUPPORT 1
-#define MEM_MODULE 1
+#include "libzstd_conf.h"
 
-#include "../contrib/zstd/compress/fse_compress.c"
-#include "../contrib/zstd/compress/huf_compress.c"
-#include "../contrib/zstd/compress/zstd_compress.c"
+#include "../contrib/zstd/lib/compress/fse_compress.c"
+#include "../contrib/zstd/lib/compress/huf_compress.c"
+
+#undef CHECK_F
+#include "../contrib/zstd/lib/compress/zstd_compress.c"
+#include "../contrib/zstd/lib/compress/zstd_compress_literals.c"
+#include "../contrib/zstd/lib/compress/zstd_compress_sequences.c"
+#include "../contrib/zstd/lib/compress/zstd_double_fast.c"
+#include "../contrib/zstd/lib/compress/zstd_fast.c"
+#include "../contrib/zstd/lib/compress/zstd_lazy.c"
+#include "../contrib/zstd/lib/compress/zstd_ldm.c"
+#include "../contrib/zstd/lib/compress/zstd_opt.c"
+#include "../contrib/zstd/lib/compress/hist.c"

data/ext/zstd_decompress.c

@@ -1,5 +1,6 @@
-#define ZSTD_LEGACY_SUPPORT 1
-#define MEM_MODULE 1
+#include "libzstd_conf.h"
 
-#include "../contrib/zstd/decompress/zstd_decompress.c"
-#include "../contrib/zstd/decompress/huf_decompress.c"
+#include "../contrib/zstd/lib/decompress/zstd_decompress.c"
+#include "../contrib/zstd/lib/decompress/zstd_decompress_block.c"
+#include "../contrib/zstd/lib/decompress/huf_decompress.c"
+#include "../contrib/zstd/lib/decompress/zstd_ddict.c"

data/ext/zstd_dictbuilder.c

@@ -1,5 +1,10 @@
-#define ZSTD_LEGACY_SUPPORT 1
-#define MEM_MODULE 1
+#include "libzstd_conf.h"
 
-#include "../contrib/zstd/dictBuilder/zdict.c"
-#include "../contrib/zstd/dictBuilder/divsufsort.c"
+#include "../contrib/zstd/lib/dictBuilder/zdict.c"
+#include "../contrib/zstd/lib/dictBuilder/divsufsort.c"
+
+#undef DISPLAY
+#undef DISPLAYLEVEL
+#undef DISPLAYUPDATE
+#define prime4bytes cover_prime4bytes
+#include "../contrib/zstd/lib/dictBuilder/cover.c"

data/ext/zstd_dictbuilder_fastcover.c

@@ -0,0 +1,3 @@
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/dictBuilder/fastcover.c"

data/ext/zstd_legacy_v01.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v01.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v01.c"

data/ext/zstd_legacy_v02.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v02.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v02.c"

data/ext/zstd_legacy_v03.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v03.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v03.c"

data/ext/zstd_legacy_v04.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v04.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v04.c"

data/ext/zstd_legacy_v05.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v05.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v05.c"

data/ext/zstd_legacy_v06.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v06.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v06.c"

data/ext/zstd_legacy_v07.c

@@ -1 +1,3 @@
-#include "../contrib/zstd/legacy/zstd_v07.c"
+#include "libzstd_conf.h"
+
+#include "../contrib/zstd/lib/legacy/zstd_v07.c"

data/gemstub.rb

@@ -1,37 +1,23 @@
-verreg = /^\s*\*\s+version:\s*(\d+(?:\.\w+)+)\s*$/i
-unless File.read("README.md", mode: "rt") =~ verreg
-  raise "``version'' is not defined or bad syntax in ``README.md''"
-end
-
-version = String($1)
+require_relative "lib/extzstd/version"
 
 GEMSTUB = Gem::Specification.new do |s|
   s.name = "extzstd"
-  s.version = version
+  s.version = Zstd::VERSION
   s.summary = "ruby bindings for Zstandard (zstd)"
   s.description = <<EOS
-unoficial ruby bindings for Zstandard (zstd) <https://github.com/Cyan4973/zstd>.
+unoficial ruby bindings for Zstandard (zstd) <https://github.com/facebook/zstd>.
 EOS
-  s.homepage = "https://osdn.jp/projects/rutsubo/"
-  s.license = "2-clause BSD License"
+  s.homepage = "https://github.com/dearblue/ruby-extzstd/"
+  s.license = "BSD-2-Clause"
   s.author = "dearblue"
   s.email = "dearblue@users.osdn.me"
 
   s.required_ruby_version = ">= 2.0"
-  s.add_development_dependency "rake"
-end
-
-LIB << "lib/extzstd/version.rb"
-
-file "lib/extzstd/version.rb" => %w(README.md) do
-  GEMSTUB.version = version
-
-  mkpath "lib/extzstd"
-  File.write "lib/extzstd/version.rb", <<-"EOS", mode: "wb"
-module Zstd
-  VERSION = #{version.inspect}
-end
-  EOS
+  s.add_development_dependency "rake", ">= 12.0"
 end
 
 EXTRA.concat(FileList["contrib/**/*"])
+
+filter = %r(contrib/zstd/(?:build|contrib|doc|examples|lib/dll|programs|tests|zlibWrapper))
+DOC.reject! { |path| path =~ filter }
+EXTRA.reject! { |path| path =~ filter }

data/lib/extzstd.rb

@@ -2,9 +2,7 @@
 
 ver = RUBY_VERSION.slice(/\d+\.\d+/)
 soname = File.basename(__FILE__, ".rb") << ".so"
-require_relative File.join(ver, soname)
-
-require_relative "extzstd/version"
+require File.join(ver, soname)
 
 require "stringio"
 
@@ -12,6 +10,40 @@ require "stringio"
 # This is ruby bindings for zstd <https://github.com/Cyan4973/zstd> the compression library.
 #
 module Zstd
+  module Internals
+    unless String.method_defined? :b
+      refine String do
+        def b
+          dup.force_encoding(Encoding::BINARY)
+        end
+      end
+    end
+  end
+
+  using Internals
+
+  refine String do
+    def to_zstd(params = nil, dict: nil)
+      ContextLess.encode(self, "".b, nil, dict, params)
+    end
+
+    def unzstd(size = nil, dict: nil)
+      Decoder.open(self, dict) { |d| return d.read(size) }
+    end
+  end
+
+  refine Object do
+    def to_zstd(params = nil, dict: nil, &block)
+      Encoder.open(self, params, dict, &block)
+    end
+
+    def unzstd(dict: nil, &block)
+      Decoder.open(self, dict, &block)
+    end
+  end
+
+  using Zstd
+
   #
   # call-seq:
   #   encode(src_string, level = nil, opts = {}) -> zstd string
@@ -26,12 +58,8 @@ module Zstd
   # [level = nil (integer or nil)]
   # [encode_params (instance of Zstd::Parameters)]
   # [opts dict: nil (string or nil)]
-  def self.encode(src, params = nil, dict: nil, &block)
-    if src.kind_of?(String)
-      return ContextLess.encode(src, Aux::EMPTY_BUFFER.dup, nil, dict, params)
-    end
-
-    Encoder.open(src, params, dict, &block)
+  def self.encode(src, *args, **opts, &block)
+    src.to_zstd(*args, **opts, &block)
   end
 
   #
@@ -40,26 +68,20 @@ module Zstd
   #   decode(zstd_stream, dict: nil) -> zstd decoder
   #   decode(zstd_stream, dict: nil) { |decoder| ... } -> yield returned value
   #
-  def self.decode(src, *args, dict: nil, &block)
-    if src.kind_of?(String)
-      case args.size
-      when 0
-        return ContextLess.decode(src, Aux::EMPTY_BUFFER.dup, nil, dict)
-      when 1
-        Decoder.open(src, dict) { |d| return d.read(args[0].to_i) }
-      else
-        raise ArgumentError, "wrong argument number (given #{args.size}, expect 1 or 2)"
-      end
-    end
-
-    unless args.empty?
-      raise ArgumentError, "wrong argument number (given #{args.size}, expect 1)"
-    end
+  def self.decode(src, *args, **opts, &block)
+    src.unzstd(*args, **opts, &block)
+  end
 
-    Decoder.open(src, dict, &block)
+  class << Zstd
+    alias compress encode
+    alias decompress decode
+    alias uncompress decode
   end
 
-  class Encoder < Struct.new(:encoder, :outport, :destbuf, :status)
+  Compressor = Encoder
+  StreamEncoder = Encoder
+
+  class Encoder
     #
     # call-seq:
     #   open(outport, level = nil, dict = nil) -> zstd encoder
@@ -73,70 +95,24 @@ module Zstd
       begin
         yield e
       ensure
-        e.close rescue nil unless e.eof?
+        e.close unless e.eof?
       end
     end
 
-    #
-    # call-seq:
-    #   initialize(outport, level = nil, dict = nil)
-    #   initialize(outport, encode_params, dict = nil)
-    #
-    # +outport+ need has +.<<+ method.
-    #
-    def initialize(outport, params = nil, dict = nil)
-      encoder = StreamEncoder.new(params, dict)
-      super encoder, outport, "".force_encoding(Encoding::BINARY), [true]
-    end
-
-    def eof
-      !status[0]
+    def self.encode(src, params = nil, dest: nil, dict: nil)
+      ContextLess.encode(src, dest || "".b, nil, dict, params)
     end
 
-    alias eof? eof
-
-    def close
-      return nil if eof?
-      encoder.end(destbuf, StreamEncoder::OUTSIZE)
-      outport << destbuf
-      status[0] = false
-      nil
-    end
-
-    def write(buf)
-      raise IOError, "closed stream" if eof?
-
-      off = 0
-      rest = buf.bytesize
-      outsize = StreamEncoder::OUTSIZE
-      while off && off < rest
-        off = encoder.update(buf, off, destbuf, outsize)
-        outport << destbuf
-      end
-
-      self
-    end
-
-    alias << write
-
-    def flush
-      raise IOError, "closed stream" if eof?
-
-      off = 0
-      encoder.flush(destbuf, StreamEncoder::OUTSIZE)
-      outport << destbuf
-
-      self
+    class << Encoder
+      alias compress encode
     end
   end
 
-  class Decoder
-    attr_reader :decoder, :inport, :readbuf, :destbuf, :status, :pos
-
-    STATUS_CLOSED = nil
-    STATUS_READY = 0
-    STATUS_INPORT_EOF = 1
+  Uncompressor = Decoder
+  Decompressor = Decoder
+  StreamDecoder = Decoder
 
+  class Decoder
     #
     # call-seq:
     #   open(inport, dict = nil) -> decoder
@@ -159,89 +135,16 @@ module Zstd
       end
     end
 
-    def initialize(inport, dict = nil)
-      raise Error, "require .read method - <%s:0x%08x>" % [inport.class, inport.object_id << 1] unless inport.respond_to?(:read)
-      @decoder = StreamDecoder.new(dict)
-      @inport = inport
-      @readbuf = StringIO.new(Aux::EMPTY_BUFFER.dup)
-      @destbuf = StringIO.new(Aux::EMPTY_BUFFER.dup)
-      @status = STATUS_READY
-      @pos = 0
-    end
-
-    def close
-      inport.close rescue nil if inport.respond_to?(:close)
-      readbuf.truncate 0
-      destbuf.truncate 0
-      @status = STATUS_CLOSED
-      nil
-    end
-
-    def eof
-      !status
-    end
-
-    alias eof? eof
-
-    def read(size = nil, dest = Aux::EMPTY_BUFFER.dup)
-      dest ||= Aux::EMPTY_BUFFER.dup
-      size &&= size.to_i
-      Aux.change_binary(dest) do
-        #dest.clear
-        dest[0 .. -1] = Aux::EMPTY_BUFFER unless dest.empty? # keep allocated heap
-        return dest unless !size || size > 0
-
-        d = Aux::EMPTY_BUFFER.dup
-        until size && size <= 0
-          if destbuf.eof?
-            break unless fetch
-          end
-
-          destbuf.read(size, d)
-          dest << d
+    def self.decode(src, dest: nil, dict: nil)
+      # NOTE: ContextLess.decode は伸長時のサイズが必要なため、常に利用できるわけではない
+      # ContextLess.decode(src, dest || "".b, nil, dict)
 
-          size -= d.bytesize if size
-        end
-      end
-
-      if dest.empty?
-        nil
-      else
-        @pos += dest.bytesize
-        dest
-      end
+      new(StringIO.new(src), dict).read(nil, dest)
     end
 
-    private
-    def fetch
-      return nil if eof?
-
-      destbuf.rewind
-      destbuf.truncate 0
-
-      while true
-        if readbuf.eof? && status != STATUS_INPORT_EOF
-          readbuf.rewind
-          unless inport.read StreamDecoder::INSIZE, readbuf.string
-            @status = STATUS_INPORT_EOF
-          end
-        end
-
-        begin
-          off = decoder.update readbuf.string, readbuf.pos, destbuf.string, StreamDecoder::OUTSIZE
-          readbuf.pos = off if off
-          return self if destbuf.size > 0
-          break if readbuf.eof? && status == STATUS_INPORT_EOF
-        rescue Zstd::InitMissingError
-          break if readbuf.eof? && status == STATUS_INPORT_EOF
-          raise
-        end
-      end
-
-      # when readbuf.eof? && status == STATUS_INPORT_EOF
-
-      @status = nil
-      nil
+    class << Decoder
+      alias decompress decode
+      alias uncompress decode
     end
   end
 
@@ -271,22 +174,4 @@ module Zstd
       end
     end
   end
-
-  module Aux
-    EMPTY_BUFFER = "".force_encoding(Encoding::BINARY).freeze
-
-    def self.io_read(io, size, buf)
-      raise Error, "encounted EOF (read error)" unless io.read(size, buf)
-      raise Error, "read size too small (read error)" unless buf.bytesize == size
-      buf
-    end
-
-    def self.change_binary(str)
-      e = str.encoding
-      str.force_encoding(Encoding::BINARY)
-      yield
-    ensure
-      str.force_encoding e rescue nil if e
-    end
-  end
 end