extzstd 0.0.1.CONCEPT → 0.0.2.CONCEPT

data/ext/extzstd.c

@@ -3,8 +3,26 @@
 VALUE mZstd;
 VALUE eError;
 
+static size_t
+aux_ZSTD_compress_nogvl(va_list *vp)
+{
+    char *dest = va_arg(*vp, char *);
+    size_t destsize = va_arg(*vp, size_t);
+    const char *src = va_arg(*vp, const char *);
+    size_t srcsize = va_arg(*vp, size_t);
+    return ZSTD_compress(dest, destsize, src, srcsize);
+}
+
+static inline size_t
+aux_ZSTD_compress(char *dest, size_t destsize, const char *src, size_t srcsize)
+{
+    return (size_t)aux_thread_call_without_gvl(
+            (void *(*)(void *))aux_ZSTD_compress_nogvl, NULL,
+            dest, destsize, src, srcsize);
+}
+
 static inline void
-zstd_encode_args(int argc, VALUE argv[], VALUE *src, VALUE *dest, size_t *maxsize)
+zstd_s_encode_args(int argc, VALUE argv[], VALUE *src, VALUE *dest, size_t *maxsize)
 {
     switch (argc) {
     case 1:
@@ -46,31 +64,50 @@ zstd_encode_args(int argc, VALUE argv[], VALUE *src, VALUE *dest, size_t *maxsiz
  *  encode(src, size, dest) -> dest with encoded string
  */
 static VALUE
-zstd_encode(int argc, VALUE argv[], VALUE mod)
+zstd_s_encode(int argc, VALUE argv[], VALUE mod)
 {
     VALUE src, dest;
     size_t maxsize;
-    zstd_encode_args(argc, argv, &src, &dest, &maxsize);
+    zstd_s_encode_args(argc, argv, &src, &dest, &maxsize);
     const char *srcp;
     size_t srcsize;
     RSTRING_GETMEM(src, srcp, srcsize);
-    size_t s = ZSTD_compress(RSTRING_PTR(dest), maxsize, srcp, srcsize);
+    rb_obj_infect(dest, src);
+    size_t s = aux_ZSTD_compress(RSTRING_PTR(dest), maxsize, srcp, srcsize);
     if (ZSTD_isError(s)) {
         rb_raise(eError,
-                 "%s:%d:%s: ZSTD_compress error - %s (%d)",
-                 __FILE__, __LINE__, __func__,
-                 ZSTD_getErrorName(s), (int)s);
+                "failed ZSTD_compress - %s(%d) in %s:%d:%s",
+                ZSTD_getErrorName(s), (int)s,
+                __FILE__, __LINE__, __func__);
     }
     if (s > maxsize) {
         rb_bug("%s:%d:%s: detect buffer overflow in ZSTD_compress - maxsize is %zd, but returned size is %zd",
-                 __FILE__, __LINE__, __func__, maxsize, s);
+                __FILE__, __LINE__, __func__, maxsize, s);
     }
     rb_str_set_len(dest, s);
     return dest;
 }
 
+static size_t
+aux_ZSTD_decompress_nogvl(va_list *vp)
+{
+    char *dest = va_arg(*vp, char *);
+    size_t destsize = va_arg(*vp, size_t);
+    const char *src = va_arg(*vp, const char *);
+    size_t srcsize = va_arg(*vp, size_t);
+    return ZSTD_decompress(dest, destsize, src, srcsize);
+}
+
+static inline size_t
+aux_ZSTD_decompress(char *dest, size_t destsize, const char *src, size_t srcsize)
+{
+    return (size_t)aux_thread_call_without_gvl(
+            (void *(*)(void *))aux_ZSTD_decompress_nogvl, NULL,
+            dest, destsize, src, srcsize);
+}
+
 static inline void
-zstd_decode_args(int argc, VALUE argv[], VALUE *src, VALUE *dest, size_t *maxsize)
+zstd_s_decode_args(int argc, VALUE argv[], VALUE *src, VALUE *dest, size_t *maxsize)
 {
     switch (argc) {
     case 2:
@@ -98,35 +135,57 @@ zstd_decode_args(int argc, VALUE argv[], VALUE *src, VALUE *dest, size_t *maxsiz
  *  decode(src, size, dest) -> dest with decoded string
  */
 static VALUE
-zstd_decode(int argc, VALUE argv[], VALUE mod)
+zstd_s_decode(int argc, VALUE argv[], VALUE mod)
 {
     VALUE src, dest;
     size_t maxsize;
-    zstd_decode_args(argc, argv, &src, &dest, &maxsize);
+    zstd_s_decode_args(argc, argv, &src, &dest, &maxsize);
     const char *srcp;
     size_t srcsize;
     RSTRING_GETMEM(src, srcp, srcsize);
-    size_t s = ZSTD_decompress(RSTRING_PTR(dest), maxsize, srcp, srcsize);
+    rb_obj_infect(dest, src);
+    size_t s = aux_ZSTD_decompress(RSTRING_PTR(dest), maxsize, srcp, srcsize);
     if (ZSTD_isError(s)) {
         rb_raise(eError,
-                 "%s:%d:%s: ZSTD_compress error - %s (%d)",
-                 __FILE__, __LINE__, __func__,
-                 ZSTD_getErrorName(s), (int)s);
+                "failed ZSTD_decompress - %s(%d) in %s:%d:%s",
+                ZSTD_getErrorName(s), (int)s,
+                __FILE__, __LINE__, __func__);
     }
     if (s > maxsize) {
         rb_bug("%s:%d:%s: detect buffer overflow in ZSTD_compress - maxsize is %zd, but returned size is %zd",
-                 __FILE__, __LINE__, __func__, maxsize, s);
+                __FILE__, __LINE__, __func__, maxsize, s);
     }
     rb_str_set_len(dest, s);
     return dest;
 }
 
+static VALUE
+libver_s_to_s(VALUE ver)
+{
+    static VALUE str;
+    if (!str) {
+        str = rb_sprintf("%d.%d.%d",
+                ZSTD_VERSION_MAJOR,
+                ZSTD_VERSION_MINOR,
+                ZSTD_VERSION_RELEASE);
+    }
+    return str;
+}
+
 void
 Init_extzstd(void)
 {
     mZstd = rb_define_module("Zstd");
-    rb_define_singleton_method(mZstd, "encode", RUBY_METHOD_FUNC(zstd_encode), -1);
-    rb_define_singleton_method(mZstd, "decode", RUBY_METHOD_FUNC(zstd_decode), -1);
+    rb_define_singleton_method(mZstd, "encode", RUBY_METHOD_FUNC(zstd_s_encode), -1);
+    rb_define_singleton_method(mZstd, "decode", RUBY_METHOD_FUNC(zstd_s_decode), -1);
+
+    VALUE libver = rb_ary_new3(3,
+            INT2FIX(ZSTD_VERSION_MAJOR),
+            INT2FIX(ZSTD_VERSION_MINOR),
+            INT2FIX(ZSTD_VERSION_RELEASE));
+    rb_define_singleton_method(libver, "to_s", RUBY_METHOD_FUNC(libver_s_to_s), 0);
+    rb_obj_freeze(libver);
+    rb_define_const(mZstd, "LIBRARY_VERSION", libver);
 
     eError = rb_define_class_under(mZstd, "Error", rb_eRuntimeError);
 

data/ext/extzstd.h

@@ -1,13 +1,15 @@
 #ifndef EXTZSTD_H
 #define EXTZSTD_H 1
 
+#include <stdarg.h>
 #include <ruby.h>
+#include <ruby/thread.h>
 #include <zstd.h>
 #include <zstd_static.h>
 
+#define RDOCFAKE(DUMMY_CODE)
+
 extern VALUE mZstd;
-extern VALUE cStreamEncoder;
-extern VALUE cStreamDecoder;
 extern VALUE eError;
 
 void init_extzstd_stream(void);
@@ -49,4 +51,29 @@ getref(VALUE v, const rb_data_type_t *type)
     return checkref(v, getrefp(v, type));
 }
 
+static inline VALUE
+aux_str_modify_expand(VALUE s, size_t z)
+{
+    rb_check_type(s, RUBY_T_STRING);
+    size_t size = RSTRING_LEN(s);
+    if (z > size) {
+        rb_str_modify_expand(s, z - size);
+    } else {
+        rb_str_modify(s);
+    }
+    return s;
+}
+
+static inline void *
+aux_thread_call_without_gvl(void *(*func)(void *), void (*cancel)(void *), ...)
+{
+    va_list va1, va2;
+    va_start(va1, cancel);
+    va_start(va2, cancel);
+    void *p = rb_thread_call_without_gvl(func, &va1, cancel, &va2);
+    va_end(va1);
+    va_end(va2);
+    return p;
+}
+
 #endif /* !EXTZSTD_H */

data/gemstub.rb

@@ -5,12 +5,12 @@ GEMSTUB = Gem::Specification.new do |s|
   s.version = Zstd::VERSION
   s.summary = "ruby bindings for Zstandard (zstd)"
   s.description = <<EOS
-ruby bindings for Zstandard (zstd) <https://github.com/Cyan4973/zstd>.
+unoficial ruby bindings for Zstandard (zstd) <https://github.com/Cyan4973/zstd>.
 EOS
-  s.homepage = "http://sourceforge.jp/projects/rutsubo/"
+  s.homepage = "https://osdn.jp/projects/rutsubo/"
   s.license = "2-clause BSD License"
   s.author = "dearblue"
-  s.email = "dearblue@users.sourceforge.jp"
+  s.email = "dearblue@users.osdn.me"
 
   s.required_ruby_version = ">= 2.0"
   s.add_development_dependency "rake", "~> 10.0"

data/lib/extzstd.rb

@@ -1,6 +1,6 @@
 #vim: set fileencoding:utf-8
 
-ver = RbConfig::CONFIG["ruby_version"]
+ver = RbConfig::CONFIG["ruby_version"].slice(/\d+\.\d+/)
 soname = File.basename(__FILE__, ".rb") << ".so"
 lib = File.join(File.dirname(__FILE__), ver, soname)
 if File.file?(lib)
@@ -10,3 +10,129 @@ else
 end
 
 require_relative "extzstd/version"
+
+require "stringio"
+
+if false
+def p(*args)
+  sf = File.basename(caller(1, 1)[0])
+  args.each do |mesg|
+    $stderr.puts "#{sf}: #{mesg.inspect}\n"
+  end
+  return *args
+end
+end
+
+module Zstd
+  module Aux
+    module_function
+    def io_read(io, size, buf)
+      raise Error, "encounted EOF (read error)" unless io.read(size, buf)
+      raise Error, "read size too small (read error)" unless buf.bytesize == size
+      buf
+    end
+  end
+
+  class Decoder < Struct.new(:decoder, :import, :readbuf, :destbuf, :status)
+    BLOCKSIZE = 1 << 18
+    STATUS_TERMINATE = nil
+    STATUS_BLOCK_APPROACH = 1
+    STATUS_INBLOCK = 2
+
+    #
+    # call-seq:
+    #   open(import) -> decoder
+    #   open(import) { |decoder| ... } -> yield returned value
+    #
+    # [import]
+    #   String instance or +read+ method haved Object.
+    #
+    def self.open(import)
+      import = StringIO.new(import) if import.kind_of?(String)
+      dec = new(import)
+
+      return dec unless block_given?
+
+      begin
+        yield(dec)
+      ensure
+        dec.close rescue nil
+      end
+    end
+
+    def initialize(import)
+      raise Error, "require .read method - <%s:0x%08x>" % [import.class, import.object_id << 1] unless import.respond_to?(:read)
+      super(LowLevelDecoder.new, import, "".b, "".b, STATUS_BLOCK_APPROACH)
+
+      # read header
+      Aux.io_read(import, decoder.next_srcsize, readbuf)
+      decoder.decode(readbuf, "", 0)
+    end
+
+    def close
+      decoder.reset
+      import.close rescue nil if import.respond_to?(:close)
+      readbuf.clear
+      destbuf.clear
+      self.status = STATUS_TERMINATE
+      nil
+    end
+
+    def eof
+      !status
+    end
+
+    alias eof? eof
+
+    def read(size = nil, dest = "".b)
+      dest.clear
+      destenc = dest.encoding
+      dest.force_encoding Encoding::BINARY
+
+      until size && size <= 0
+        if destbuf.empty?
+          unless fetch
+            return nil if dest.empty?
+            break
+          end
+        end
+
+        d = destbuf.slice!(0, size || destbuf.bytesize)
+        dest << d
+
+        size -= d.bytesize if size
+      end
+
+      dest
+    ensure
+      dest.force_encoding destenc rescue nil if destenc
+    end
+
+    private
+    def fetch
+      return nil if eof?
+
+      while true
+        if status == STATUS_INBLOCK
+          s = decoder.next_srcsize
+          if s > 0
+            Aux.io_read(import, s, readbuf)
+            return destbuf if decoder.decode(readbuf, destbuf, BLOCKSIZE)
+            next
+          end
+          self.status = STATUS_BLOCK_APPROACH
+        end
+
+        # status == STATUS_BLOCK_APPROACH
+        s = decoder.next_srcsize
+        if s == 0
+          self.status = STATUS_TERMINATE
+          return nil
+        end
+        Aux.io_read(import, s, readbuf)
+        decoder.decode(readbuf, "", 0)
+        self.status = STATUS_INBLOCK
+      end
+    end
+  end
+end

data/lib/extzstd/version.rb

@@ -1,5 +1,3 @@
-require "rubygems"
-
 module Zstd
-  VERSION = Gem::Version.new("0.0.1.CONCEPT")
+  VERSION = "0.0.2.CONCEPT"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: extzstd
 version: !ruby/object:Gem::Version
-  version: 0.0.1.CONCEPT
+  version: 0.0.2.CONCEPT
 platform: ruby
 authors:
 - dearblue
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-26 00:00:00.000000000 Z
+date: 2015-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -25,14 +25,15 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '10.0'
 description: |
-  ruby bindings for Zstandard (zstd) <https://github.com/Cyan4973/zstd>.
-email: dearblue@users.sourceforge.jp
+  unoficial ruby bindings for Zstandard (zstd) <https://github.com/Cyan4973/zstd>.
+email: dearblue@users.osdn.me
 executables: []
 extensions:
 - ext/extconf.rb
 extra_rdoc_files:
 - LICENSE
 - README.md
+- contrib/zstd/LICENSE
 - ext/extzstd-stream.c
 - ext/extzstd.c
 - ext/extzstd.h
@@ -58,12 +59,16 @@ files:
 - gemstub.rb
 - lib/extzstd.rb
 - lib/extzstd/version.rb
-homepage: http://sourceforge.jp/projects/rutsubo/
+homepage: https://osdn.jp/projects/rutsubo/
 licenses:
 - 2-clause BSD License
 metadata: {}
 post_install_message: 
-rdoc_options: []
+rdoc_options:
+- "--charset"
+- UTF-8
+- "-m"
+- README.md
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
@@ -78,8 +83,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: ruby bindings for Zstandard (zstd)
 test_files: []
+has_rdoc: