extzstd 0.0.1.CONCEPT → 0.0.2.CONCEPT

data/contrib/zstd/zstd.h

@@ -46,9 +46,9 @@ extern "C" {
 *  Version
 **************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
-#define ZSTD_VERSION_MINOR    0    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
-#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
+#define ZSTD_VERSION_MINOR    1    /* for new (non-breaking) interface capabilities */
+#define ZSTD_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 unsigned ZSTD_versionNumber (void);
 
 
@@ -64,13 +64,13 @@ size_t ZSTD_decompress( void* dst, size_t maxOriginalSize,
 /*
 ZSTD_compress() :
     Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'.
-    Destination buffer should be sized to handle worst cases situations (input data not compressible).
-    Worst case size evaluation is provided by function ZSTD_compressBound().
+    Destination buffer must be already allocated.
+    Compression runs faster if maxDstSize >=  ZSTD_compressBound(srcSize).
     return : the number of bytes written into buffer 'dst'
              or an error code if it fails (which can be tested using ZSTD_isError())
 
 ZSTD_decompress() :
-    compressedSize : is obviously the source size
+    compressedSize : is the exact source size
     maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.
                       It must be equal or larger than originalSize, otherwise decompression will fail.
     return : the number of bytes decompressed into destination buffer (originalSize)
@@ -81,22 +81,13 @@ ZSTD_decompress() :
 /**************************************
 *  Tool functions
 **************************************/
-size_t      ZSTD_compressBound(size_t srcSize);   /* maximum compressed size */
+size_t      ZSTD_compressBound(size_t srcSize);   /* maximum compressed size (worst case scenario) */
 
 /* Error Management */
 unsigned    ZSTD_isError(size_t code);         /* tells if a return value is an error code */
 const char* ZSTD_getErrorName(size_t code);    /* provides error code string (useful for debugging) */
 
 
-#define CHECK_OVERFLOW(statment) \
-    do { \
-        if (!(statment)) { \
-            fprintf(stderr, \
-                    "%s:%d:%s: detect buffer overflow - assert for (%s)\n", \
-                    __FILE__, __LINE__, __func__, #statment); \
-        } \
-    } while (0) \
-
 #if defined (__cplusplus)
 }
 #endif

data/contrib/zstd/zstd_static.h

@@ -45,30 +45,39 @@ extern "C" {
 /**************************************
 *  Streaming functions
 **************************************/
-typedef void* ZSTD_cctx_t;
-ZSTD_cctx_t ZSTD_createCCtx(void);
-size_t      ZSTD_freeCCtx(ZSTD_cctx_t cctx);
+typedef struct ZSTD_Cctx_s ZSTD_Cctx;
+ZSTD_Cctx* ZSTD_createCCtx(void);
+size_t     ZSTD_freeCCtx(ZSTD_Cctx* cctx);
 
-size_t ZSTD_compressBegin(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize);
-size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
-size_t ZSTD_compressEnd(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize);
+size_t ZSTD_compressBegin(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
+size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
 
-typedef void* ZSTD_dctx_t;
-ZSTD_dctx_t ZSTD_createDCtx(void);
-size_t      ZSTD_freeDCtx(ZSTD_dctx_t dctx);
 
-size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx);
-size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+typedef struct ZSTD_Dctx_s ZSTD_Dctx;
+ZSTD_Dctx* ZSTD_createDCtx(void);
+size_t     ZSTD_resetDCtx(ZSTD_Dctx* dctx);
+size_t     ZSTD_freeDCtx(ZSTD_Dctx* dctx);
 
+size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx);
+size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
+/*
+  Use above functions alternatively.
+  ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as input to ZSTD_decompressContinue().
+  This value is expected to be provided, precisely, as 'srcSize'.
+  Otherwise, compression will fail (result is an error code, which can be tested using ZSTD_isError() )
+  ZSTD_decompressContinue() result is the number of bytes regenerated within 'dst'.
+  It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
+*/
 
 /**************************************
 *  Error management
 **************************************/
 #define ZSTD_LIST_ERRORS(ITEM) \
         ITEM(ZSTD_OK_NoError) ITEM(ZSTD_ERROR_GENERIC) \
-        ITEM(ZSTD_ERROR_wrongMagicNumber) \
-        ITEM(ZSTD_ERROR_wrongSrcSize) ITEM(ZSTD_ERROR_maxDstSize_tooSmall) \
-        ITEM(ZSTD_ERROR_wrongLBlockSize) \
+        ITEM(ZSTD_ERROR_MagicNumber) \
+        ITEM(ZSTD_ERROR_SrcSize) ITEM(ZSTD_ERROR_maxDstSize_tooSmall) \
+        ITEM(ZSTD_ERROR_corruption) \
         ITEM(ZSTD_ERROR_maxCode)
 
 #define ZSTD_GENERATE_ENUM(ENUM) ENUM,
@@ -77,4 +86,4 @@ typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes;   /* expo
 
 #if defined (__cplusplus)
 }
-#endif
+#endif

data/ext/extconf.rb

@@ -7,11 +7,16 @@ dir = File.dirname(__FILE__).gsub(/[\[\{\?\*]/, "[\\0]")
 filepattern = "{.,../contrib/zstd}/*.c"
 target = File.join(dir, filepattern)
 files = Dir.glob(target).map { |n| File.basename n }
+## reject fse.c, because it's included in zstd.c
 files.reject! { |n| "/contrib/zstd/fse.c".include?(n) }
 $srcs = files
 
 $VPATH.push "$(srcdir)/../contrib/zstd"
 
-find_header "zstd.h", "$(srcdir)/../contrib/zstd"
+find_header "zstd.h", "$(srcdir)/../contrib/zstd" or abort 1
+
+if RbConfig::CONFIG["arch"] =~ /mingw/
+  $LDFLAGS << " -static-libgcc"
+end
 
 create_makefile("extzstd")

data/ext/extzstd-stream.c

@@ -1,30 +1,53 @@
 #include "extzstd.h"
 
-VALUE cEncoder;
-ID id_op_lshift;
+static ID id_op_lshift;
 
-struct zstdencoder
+#if 0
+static size_t
+aux_read(VALUE io, size_t size, char *buf)
+{
+    VALUE w = SEND(io, id_read, SIZET2NUM(size));
+    if (NIL_P(w)) { return 0; }
+    rb_check_type(w, RUBY_T_STRING);
+    size_t s = RSTRING_LEN(w);
+    if (s > size) {
+        rb_raise(rb_eRuntimeError,
+                "read size is too big - #<%s:%p> (size is %"PRIuSIZE" for %"PRIuSIZE")",
+                rb_obj_classname(io), (void *)io, s, size);
+    }
+    memcpy(buf, RSTRING_PTR(w), s);
+    return s;
+}
+#endif
+
+/*
+ * class Zstd::Encoder
+ */
+
+static VALUE cEncoder;
+
+struct encoder
 {
     VALUE outport;
     VALUE destbuf;
-    ZSTD_cctx_t encoder;
+    ZSTD_Cctx *encoder;
 };
 
 static void
-zstdenc_mark(void *pp)
+enc_mark(void *pp)
 {
     if (pp) {
-        struct zstdencoder *p = pp;
+        struct encoder *p = pp;
         rb_gc_mark(p->outport);
         rb_gc_mark(p->destbuf);
     }
 }
 
 static void
-zstdenc_free(void *pp)
+enc_free(void *pp)
 {
     if (pp) {
-        struct zstdencoder *p = pp;
+        struct encoder *p = pp;
         if (p->encoder) {
             ZSTD_freeCCtx(p->encoder);
         }
@@ -35,39 +58,39 @@ zstdenc_free(void *pp)
     }
 }
 
-static const rb_data_type_t zstdencoder_type = {
+static const rb_data_type_t encoder_type = {
     .wrap_struct_name = "extzstd.Zstd.Encoder",
-    .function.dmark = zstdenc_mark,
-    .function.dfree = zstdenc_free,
-    /* .function.dsize = zstdenc_size, */
+    .function.dmark = enc_mark,
+    .function.dfree = enc_free,
+    /* .function.dsize = enc_size, */
 };
 
 
 static VALUE
-zstdenc_alloc(VALUE klass)
+enc_alloc(VALUE klass)
 {
-    struct zstdencoder *p;
-    VALUE v = TypedData_Make_Struct(klass, struct zstdencoder, &zstdencoder_type, p);
+    struct encoder *p;
+    VALUE v = TypedData_Make_Struct(klass, struct encoder, &encoder_type, p);
     p->outport = Qnil;
     p->destbuf = Qnil;
     p->encoder = NULL;
     return v;
 }
 
-static inline struct zstdencoder *
+static inline struct encoder *
 getencoderp(VALUE enc)
 {
-    return getrefp(enc, &zstdencoder_type);
+    return getrefp(enc, &encoder_type);
 }
 
-static inline struct zstdencoder *
+static inline struct encoder *
 getencoder(VALUE enc)
 {
-    return getref(enc, &zstdencoder_type);
+    return getref(enc, &encoder_type);
 }
 
 static inline void
-zstdenc_init_args(int argc, VALUE argv[], VALUE *outport)
+enc_init_args(int argc, VALUE argv[], VALUE *outport)
 {
     rb_scan_args(argc, argv, "01", outport);
     if (NIL_P(*outport)) {
@@ -78,30 +101,31 @@ zstdenc_init_args(int argc, VALUE argv[], VALUE *outport)
 /*
  * call-seq:
  *  initialize(outport) -> self
+ *
+ * +outport+ is required +.<<+ method for behavior as same as +IO#<<+ method.
  */
 static VALUE
-zstdenc_init(int argc, VALUE argv[], VALUE senc)
+enc_init(int argc, VALUE argv[], VALUE enc)
 {
-    struct zstdencoder *p = getencoder(senc);
-    if (p->encoder) { reiniterror(senc); }
-    zstdenc_init_args(argc, argv, &p->outport);
+    struct encoder *p = getencoder(enc);
+    if (p->encoder) { reiniterror(enc); }
+    enc_init_args(argc, argv, &p->outport);
     p->destbuf = rb_str_buf_new(4 * 1024); /* FIXME: ``4 * 1024`` to constant value */
     p->encoder = ZSTD_createCCtx();
     if (!p->encoder) {
         rb_raise(eError,
-                 "failed ZSTD_createCCtx()");
+                "failed ZSTD_createCCtx()");
     }
 
     size_t destsize = rb_str_capacity(p->destbuf);
-    //rb_str_resize(p->destbuf, destsize); /* FIXME: rb_str_resize は縮小もするので、できれば rb_str_modify_expand をつかう */
 
     size_t s = ZSTD_compressBegin(p->encoder, RSTRING_PTR(p->destbuf), destsize);
     if (ZSTD_isError(s)) {
         rb_raise(eError,
-                 "failed ZSTD_compressBegin() - %s (%d)",
-                 ZSTD_getErrorName(s), (int)s);
+                "failed ZSTD_compressBegin() - %s (%d)",
+                ZSTD_getErrorName(s), (int)s);
     }
-    
+
     if (s > destsize) {
         rb_bug("detect buffer overflow from in ZSTD_compressBegin() (destbuf.bytesize=%d, returned bytesize=%d)",
                (int)RSTRING_LEN(p->destbuf), (int)s);
@@ -110,7 +134,7 @@ zstdenc_init(int argc, VALUE argv[], VALUE senc)
     rb_str_set_len(p->destbuf, s);
     rb_funcall2(p->outport, id_op_lshift, 1, &p->destbuf);
 
-    return senc;
+    return enc;
 }
 
 /*
@@ -118,10 +142,10 @@ zstdenc_init(int argc, VALUE argv[], VALUE senc)
  *  write(src) -> self
  */
 static VALUE
-zstdenc_write(int argc, VALUE argv[], VALUE senc)
+enc_write(int argc, VALUE argv[], VALUE enc)
 {
-    struct zstdencoder *p = getencoder(senc);
-    if (!p->encoder) { referror(senc); }
+    struct encoder *p = getencoder(enc);
+    if (!p->encoder) { referror(enc); }
 
     VALUE src;
     rb_scan_args(argc, argv, "1", &src);
@@ -131,14 +155,15 @@ zstdenc_write(int argc, VALUE argv[], VALUE senc)
     rb_str_modify(p->destbuf);
     rb_str_set_len(p->destbuf, 0);
     rb_str_modify_expand(p->destbuf, destsize);
-
+    rb_obj_infect(enc, src);
+    rb_obj_infect(p->destbuf, enc);
     size_t s = ZSTD_compressContinue(p->encoder, RSTRING_PTR(p->destbuf), destsize, RSTRING_PTR(src), RSTRING_LEN(src));
     if (ZSTD_isError(s)) {
         rb_raise(eError,
-                 "failed ZSTD_compressContinue() - %s (%d)",
-                 ZSTD_getErrorName(s), (int)s);
+                "failed ZSTD_compressContinue() - %s (%d)",
+                ZSTD_getErrorName(s), (int)s);
     }
-    
+
     if (s > destsize) {
         rb_bug("detect buffer overflow from in ZSTD_compressContinue() (destbuf.bytesize=%d, returned bytesize=%d)",
                (int)RSTRING_LEN(p->destbuf), (int)s);
@@ -147,7 +172,7 @@ zstdenc_write(int argc, VALUE argv[], VALUE senc)
     rb_str_set_len(p->destbuf, s);
     rb_funcall2(p->outport, id_op_lshift, 1, &p->destbuf);
 
-    return senc;
+    return enc;
 }
 
 /*
@@ -155,10 +180,10 @@ zstdenc_write(int argc, VALUE argv[], VALUE senc)
  *  finish -> self
  */
 static VALUE
-zstdenc_finish(int argc, VALUE argv[], VALUE senc)
+enc_finish(int argc, VALUE argv[], VALUE enc)
 {
-    struct zstdencoder *p = getencoder(senc);
-    if (!p->encoder) { referror(senc); }
+    struct encoder *p = getencoder(enc);
+    if (!p->encoder) { referror(enc); }
 
     rb_scan_args(argc, argv, "0");
 
@@ -170,8 +195,8 @@ zstdenc_finish(int argc, VALUE argv[], VALUE senc)
     size_t s = ZSTD_compressEnd(p->encoder, RSTRING_PTR(p->destbuf), destsize);
     if (ZSTD_isError(s)) {
         rb_raise(eError,
-                 "failed ZSTD_compressEnd() - %s (%d)",
-                 ZSTD_getErrorName(s), (int)s);
+                "failed ZSTD_compressEnd() - %s (%d)",
+                ZSTD_getErrorName(s), (int)s);
     }
 
     if (s > destsize) {
@@ -182,38 +207,192 @@ zstdenc_finish(int argc, VALUE argv[], VALUE senc)
     rb_str_set_len(p->destbuf, s);
     rb_funcall2(p->outport, id_op_lshift, 1, &p->destbuf);
 
-    return senc;
+    return enc;
 }
 
 static VALUE
-zstdenc_getoutport(VALUE enc)
+enc_getoutport(VALUE enc)
 {
     return getencoder(enc)->outport;
 }
 
 static VALUE
-zstdenc_setoutport(VALUE enc, VALUE outport)
+enc_setoutport(VALUE enc, VALUE outport)
 {
     return getencoder(enc)->outport = outport;
 }
 
+/*
+ * class Zstd::LowLevelDecoder
+ */
+
+static VALUE cLLDecoder;
+
+struct lldecoder
+{
+    ZSTD_Dctx *decoder;
+};
+
+static void
+lldec_mark(void *pp)
+{
+    if (pp) {
+        struct lldecoder *p = pp;
+    }
+}
+
+static void
+lldec_free(void *pp)
+{
+    if (pp) {
+        struct lldecoder *p = pp;
+        if (p->decoder) {
+            ZSTD_freeDCtx(p->decoder);
+        }
+        free(p);
+    }
+}
+
+static const rb_data_type_t lldecoder_type = {
+    .wrap_struct_name = "extzstd.Zstd.LowLevelDecoder",
+    .function.dmark = lldec_mark,
+    .function.dfree = lldec_free,
+    /* .function.dsize = lldec_size, */
+};
+
+
+static VALUE
+lldec_alloc(VALUE klass)
+{
+    struct lldecoder *p;
+    VALUE v = TypedData_Make_Struct(klass, struct lldecoder, &lldecoder_type, p);
+    p->decoder = NULL;
+    return v;
+}
+
+static inline struct lldecoder *
+getlldecoderp(VALUE v)
+{
+    return getrefp(v, &lldecoder_type);
+}
+
+static inline struct lldecoder *
+getlldecoder(VALUE v)
+{
+    return getref(v, &lldecoder_type);
+}
+
+/*
+ * call-seq:
+ *  initialize
+ */
+static VALUE
+lldec_init(int argc, VALUE argv[], VALUE dec)
+{
+    struct lldecoder *p = getlldecoder(dec);
+    if (p->decoder) { reiniterror(dec); }
+    rb_check_arity(argc, 0, 0);
+    p->decoder = ZSTD_createDCtx();
+    if (!p->decoder) {
+        rb_raise(eError,
+                "failed ZSTD_createDCtx()");
+    }
+
+    ZSTD_resetDCtx(p->decoder);
+
+    return dec;
+}
+
+/*
+ * call-seq:
+ *  reset -> self
+ */
+static VALUE
+lldec_reset(VALUE dec)
+{
+    struct lldecoder *p = getlldecoder(dec);
+    if (!p->decoder) { referror(dec); }
+    size_t s = ZSTD_resetDCtx(p->decoder);
+    if (ZSTD_isError(s)) {
+        rb_raise(eError,
+                "failed ZSTD_resetDCtx() - %s (%d)",
+                ZSTD_getErrorName(s), (int)s);
+    }
+    return dec;
+}
+
+/*
+ * call-seq:
+ *  next_srcsize -> next size as integer
+ */
+static VALUE
+lldec_next_srcsize(VALUE dec)
+{
+    struct lldecoder *p = getlldecoder(dec);
+    if (!p->decoder) { referror(dec); }
+    size_t s = ZSTD_nextSrcSizeToDecompress(p->decoder);
+    if (ZSTD_isError(s)) {
+        rb_raise(eError,
+                "failed ZSTD_nextSrcSizeToDecompress() - %s (%d)",
+                ZSTD_getErrorName(s), (int)s);
+    }
+    return SIZET2NUM(s);
+}
+
+/*
+ * call-seq:
+ *  decode(src, dest, maxdestsize) -> dest OR nil
+ */
+static VALUE
+lldec_decode(VALUE dec, VALUE src, VALUE dest, VALUE maxdestsize)
+{
+    struct lldecoder *p = getlldecoder(dec);
+    if (!p->decoder) { referror(dec); }
+
+    rb_obj_infect(dec, src);
+    rb_obj_infect(dest, dec);
+    const char *srcp = StringValuePtr(src);
+    size_t srcsize = RSTRING_LEN(src);
+    size_t destsize = NUM2UINT(maxdestsize); /* サイズ制限のために NUM2UINT にしている */
+    aux_str_modify_expand(dest, destsize);
+    char *destp = RSTRING_PTR(dest);
+    size_t s = ZSTD_decompressContinue(p->decoder, destp, destsize, srcp, srcsize);
+    if (ZSTD_isError(s)) {
+        rb_raise(eError,
+                "failed ZSTD_decompressContinue() - %s (%d)",
+                ZSTD_getErrorName(s), (int)s);
+    }
+    rb_str_set_len(dest, s);
+    if (s == 0) {
+        return Qnil;
+    } else {
+        return dest;
+    }
+}
+
+/*
+ * initializer for Zstd::Encoder and Zstd::LowLevelDecoder
+ */
+
 void
 init_extzstd_stream(void)
 {
     id_op_lshift = rb_intern("<<");
 
+    RDOCFAKE(mZstd = rb_define_module("Zstd"));
+
     cEncoder = rb_define_class_under(mZstd, "Encoder", rb_cObject);
-    rb_define_alloc_func(cEncoder, zstdenc_alloc);
-    rb_define_method(cEncoder, "initialize", RUBY_METHOD_FUNC(zstdenc_init), -1);
-    rb_define_method(cEncoder, "write", RUBY_METHOD_FUNC(zstdenc_write), -1);
-    rb_define_method(cEncoder, "finish", RUBY_METHOD_FUNC(zstdenc_finish), -1);
-    rb_define_method(cEncoder, "outport", RUBY_METHOD_FUNC(zstdenc_getoutport), 0);
-    rb_define_method(cEncoder, "outport=", RUBY_METHOD_FUNC(zstdenc_setoutport), 1);
-/*
-    cStreamDecoder = rb_define_class_under(mZstd, "StreamDecoder", rb_cObject);
-    rb_define_alloc_func(cStreamDecoder, zstdenc_alloc);
-    rb_define_method(cStreamDecoder, "initialize", RUBY_METHOD_FUNC(ext_sdec_init), -1);
-    rb_define_method(cStreamDecoder, "read", RUBY_METHOD_FUNC(ext_sdec_read), -1);
-    rb_define_method(cStreamDecoder, "finish", RUBY_METHOD_FUNC(ext_sdec_finish), -1);
-*/
+    rb_define_alloc_func(cEncoder, enc_alloc);
+    rb_define_method(cEncoder, "initialize", RUBY_METHOD_FUNC(enc_init), -1);
+    rb_define_method(cEncoder, "write", RUBY_METHOD_FUNC(enc_write), -1);
+    rb_define_method(cEncoder, "finish", RUBY_METHOD_FUNC(enc_finish), -1);
+    rb_define_method(cEncoder, "outport", RUBY_METHOD_FUNC(enc_getoutport), 0);
+    rb_define_method(cEncoder, "outport=", RUBY_METHOD_FUNC(enc_setoutport), 1);
+
+    cLLDecoder = rb_define_class_under(mZstd, "LowLevelDecoder", rb_cObject);
+    rb_define_alloc_func(cLLDecoder, lldec_alloc);
+    rb_define_method(cLLDecoder, "initialize", RUBY_METHOD_FUNC(lldec_init), -1);
+    rb_define_method(cLLDecoder, "reset", RUBY_METHOD_FUNC(lldec_reset), 0);
+    rb_define_method(cLLDecoder, "next_srcsize", RUBY_METHOD_FUNC(lldec_next_srcsize), 0);
+    rb_define_method(cLLDecoder, "decode", RUBY_METHOD_FUNC(lldec_decode), 3);
 }