extzstd 0.0.2.CONCEPT → 0.0.3.CONCEPT

data/contrib/zstd/legacy/zstd_v06.h

@@ -0,0 +1,185 @@
+/*
+    zstd_v06 - decoder for 0.6 format
+    Header File
+    Copyright (C) 2014-2016, Yann Collet.
+
+    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+    * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following disclaimer
+    in the documentation and/or other materials provided with the
+    distribution.
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+    You can contact the author at :
+    - zstd source repository : https://github.com/Cyan4973/zstd
+*/
+#ifndef ZSTDv06_H
+#define ZSTDv06_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-*************************************
+*  Dependencies
+***************************************/
+#include <stddef.h>   /* size_t */
+
+
+/*-***************************************************************
+*  Export parameters
+*****************************************************************/
+/*!
+*  ZSTDv06_DLL_EXPORT :
+*  Enable exporting of functions when building a Windows DLL
+*/
+#if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1)
+#  define ZSTDLIB_API __declspec(dllexport)
+#else
+#  define ZSTDLIB_API
+#endif
+
+
+/* *************************************
+*  Simple functions
+***************************************/
+/*! ZSTDv06_decompress() :
+    `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail.
+    `dstCapacity` must be large enough, equal or larger than originalSize.
+    @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+              or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */
+ZSTDLIB_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
+                              const void* src, size_t compressedSize);
+
+
+/* *************************************
+*  Helper functions
+***************************************/
+ZSTDLIB_API size_t      ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */
+
+/* Error Management */
+ZSTDLIB_API unsigned    ZSTDv06_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
+ZSTDLIB_API const char* ZSTDv06_getErrorName(size_t code);     /*!< provides readable string for an error code */
+
+
+/* *************************************
+*  Explicit memory management
+***************************************/
+/** Decompression context */
+typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx;
+ZSTDLIB_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void);
+ZSTDLIB_API size_t     ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx);      /*!< @return : errorCode */
+
+/** ZSTDv06_decompressDCtx() :
+*   Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */
+ZSTDLIB_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+
+/*-***********************
+*  Dictionary API
+*************************/
+/*! ZSTDv06_decompress_usingDict() :
+*   Decompression using a pre-defined Dictionary content (see dictBuilder).
+*   Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
+*   Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */
+ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
+                                             void* dst, size_t dstCapacity,
+                                       const void* src, size_t srcSize,
+                                       const void* dict,size_t dictSize);
+
+
+/*-************************
+*  Advanced Streaming API
+***************************/
+struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; };
+typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams;
+
+ZSTDLIB_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input */
+ZSTDLIB_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIB_API void   ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx);
+
+ZSTDLIB_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx);
+ZSTDLIB_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+
+
+/* *************************************
+*  ZBUFF API
+***************************************/
+
+typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx;
+ZSTDLIB_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void);
+ZSTDLIB_API size_t      ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx);
+
+ZSTDLIB_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx);
+ZSTDLIB_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize);
+
+ZSTDLIB_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx,
+                                            void* dst, size_t* dstCapacityPtr,
+                                      const void* src, size_t* srcSizePtr);
+
+/*-***************************************************************************
+*  Streaming decompression howto
+*
+*  A ZBUFFv06_DCtx object is required to track streaming operations.
+*  Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources.
+*  Use ZBUFFv06_decompressInit() to start a new decompression operation,
+*   or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary.
+*  Note that ZBUFFv06_DCtx objects can be re-init multiple times.
+*
+*  Use ZBUFFv06_decompressContinue() repetitively to consume your input.
+*  *srcSizePtr and *dstCapacityPtr can be any size.
+*  The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
+*  Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
+*  The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
+*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
+*            or 0 when a frame is completely decoded,
+*            or an error code, which can be tested using ZBUFFv06_isError().
+*
+*  Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize()
+*  output : ZBUFFv06_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
+*  input  : ZBUFFv06_recommendedDInSize == 128KB + 3;
+*           just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
+* *******************************************************************************/
+
+
+/* *************************************
+*  Tool functions
+***************************************/
+ZSTDLIB_API unsigned ZBUFFv06_isError(size_t errorCode);
+ZSTDLIB_API const char* ZBUFFv06_getErrorName(size_t errorCode);
+
+/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
+*   These sizes are just hints, they tend to offer better latency */
+ZSTDLIB_API size_t ZBUFFv06_recommendedDInSize(void);
+ZSTDLIB_API size_t ZBUFFv06_recommendedDOutSize(void);
+
+
+/*-*************************************
+*  Constants
+***************************************/
+#define ZSTDv06_MAGICNUMBER 0xFD2FB526   /* v0.6 */
+
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif  /* ZSTDv06_BUFFERED_H */

data/ext/extconf.rb

@@ -1,22 +1,20 @@
 #!ruby
-#vim: set fileencoding:utf-8
 
 require "mkmf"
 
-dir = File.dirname(__FILE__).gsub(/[\[\{\?\*]/, "[\\0]")
-filepattern = "{.,../contrib/zstd}/*.c"
-target = File.join(dir, filepattern)
-files = Dir.glob(target).map { |n| File.basename n }
-## reject fse.c, because it's included in zstd.c
-files.reject! { |n| "/contrib/zstd/fse.c".include?(n) }
-$srcs = files
+find_header "zstd.h", "$(srcdir)/../contrib/zstd/common" or abort "can't find ``zstd.h''"
+find_header "zdict.h", "$(srcdir)/../contrib/zstd/dictBuilder" or abort "can't find ``zdict.h''"
+find_header "zstd_legacy.h", "$(srcdir)/../contrib/zstd/legacy" or abort "can't find ``zstd_legacy.h''"
 
-$VPATH.push "$(srcdir)/../contrib/zstd"
-
-find_header "zstd.h", "$(srcdir)/../contrib/zstd" or abort 1
+#dir = File.dirname(__FILE__).gsub(/[\[\{\?\*]/, "[\\0]")
+#filepattern = "{.,../contrib/zstd}/**/*.c"
+#target = File.join(dir, filepattern)
+#files = Dir.glob(target).sort.map { |n| File.basename n }
+#$srcs = files
+#$VPATH.push "$(srcdir)/../contrib/zstd", "$(srcdir)/../contrib/zstd/legacy"
 
 if RbConfig::CONFIG["arch"] =~ /mingw/
   $LDFLAGS << " -static-libgcc"
 end
 
-create_makefile("extzstd")
+create_makefile File.join(RUBY_VERSION.slice(/\d+\.\d+/), "extzstd")

data/ext/extzstd.c

@@ -1,193 +1,546 @@
 #include "extzstd.h"
+#include <mem.h>
+#include <error_public.h>
+#include <zdict.h>
 
-VALUE mZstd;
-VALUE eError;
+VALUE extzstd_mZstd;
 
-static size_t
-aux_ZSTD_compress_nogvl(va_list *vp)
+/*
+ * constant Zstd::LIBRARY_VERSION
+ */
+
+static VALUE
+libver_s_to_i(VALUE ver)
 {
-    char *dest = va_arg(*vp, char *);
-    size_t destsize = va_arg(*vp, size_t);
-    const char *src = va_arg(*vp, const char *);
-    size_t srcsize = va_arg(*vp, size_t);
-    return ZSTD_compress(dest, destsize, src, srcsize);
+    return UINT2NUM(ZSTD_VERSION_NUMBER);
 }
 
-static inline size_t
-aux_ZSTD_compress(char *dest, size_t destsize, const char *src, size_t srcsize)
+static VALUE
+libver_s_to_s(VALUE ver)
 {
-    return (size_t)aux_thread_call_without_gvl(
-            (void *(*)(void *))aux_ZSTD_compress_nogvl, NULL,
-            dest, destsize, src, srcsize);
+    static VALUE str;
+    if (!str) {
+        str = rb_sprintf("%d.%d.%d",
+                ZSTD_VERSION_MAJOR,
+                ZSTD_VERSION_MINOR,
+                ZSTD_VERSION_RELEASE);
+        rb_obj_freeze(str);
+    }
+    return str;
 }
 
-static inline void
-zstd_s_encode_args(int argc, VALUE argv[], VALUE *src, VALUE *dest, size_t *maxsize)
+static void
+init_libver(void)
 {
-    switch (argc) {
-    case 1:
-        *src = argv[0];
-        rb_check_type(*src, RUBY_T_STRING);
-        *maxsize = ZSTD_compressBound(RSTRING_LEN(*src));
-        *dest = rb_str_buf_new(*maxsize);
-        return;
-    case 2:
-        *src = argv[0];
-        rb_check_type(*src, RUBY_T_STRING);
-        *dest = argv[1];
-        if (rb_type_p(*dest, RUBY_T_STRING)) {
-            *maxsize = ZSTD_compressBound(RSTRING_LEN(*src));
-            rb_str_resize(*dest, *maxsize);
-        } else {
-            *maxsize = NUM2SIZET(*dest);
-            *dest = rb_str_buf_new(*maxsize);
-        }
-        return;
-    case 3:
-        *src = argv[0];
-        rb_check_type(*src, RUBY_T_STRING);
-        *maxsize = NUM2SIZET(argv[1]);
-        *dest = argv[2];
-        rb_check_type(*dest, RUBY_T_STRING);
-        rb_str_resize(*dest, *maxsize);
-        return;
+    VALUE libver = AUX_TUPLE(
+            INT2FIX(ZSTD_VERSION_MAJOR),
+            INT2FIX(ZSTD_VERSION_MINOR),
+            INT2FIX(ZSTD_VERSION_RELEASE));
+    rb_define_singleton_method(libver, "to_i", RUBY_METHOD_FUNC(libver_s_to_i), 0);
+    rb_define_singleton_method(libver, "to_s", RUBY_METHOD_FUNC(libver_s_to_s), 0);
+    rb_define_singleton_method(libver, "to_str", RUBY_METHOD_FUNC(libver_s_to_s), 0);
+    rb_obj_freeze(libver);
+    rb_define_const(extzstd_mZstd, "LIBRARY_VERSION", libver);
+}
+
+/*
+ * error classes
+ */
+
+/*
+ * Document-class: Zstd::Exceptions
+ *
+ * Catch extzstd errors for rescue statment.
+ */
+VALUE extzstd_mExceptions;
+
+VALUE extzstd_eError;
+VALUE extzstd_eError;
+VALUE extzstd_eGenericError;
+VALUE extzstd_ePrefixUnknownError;
+VALUE extzstd_eFrameParameterUnsupportedError;
+VALUE extzstd_eFrameParameterUnsupportedBy32bitsError;
+VALUE extzstd_eCompressionParameterUnsupportedError;
+VALUE extzstd_eInitMissingError;
+VALUE extzstd_eMemoryAllocationError;
+VALUE extzstd_eStageWrongError;
+VALUE extzstd_eDstSizeTooSmallError;
+VALUE extzstd_eSrcSizeWrongError;
+VALUE extzstd_eCorruptionDetectedError;
+VALUE extzstd_eChecksumWrongError;
+VALUE extzstd_eTableLogTooLargeError;
+VALUE extzstd_eMaxSymbolValueTooLargeError;
+VALUE extzstd_eMaxSymbolValueTooSmallError;
+VALUE extzstd_eDictionaryCorruptedError;
+VALUE extzstd_eDictionaryWrongError;
+
+
+void
+extzstd_check_error(ssize_t errcode)
+{
+    if (ZSTD_isError(errcode)) {
+        extzstd_error(errcode);
+    }
+}
+
+void
+extzstd_error(ssize_t errcode)
+{
+    rb_exc_raise(extzstd_make_error(errcode));
+}
+
+VALUE
+extzstd_make_errorf(VALUE exc, const char *fmt, ...)
+{
+    if (fmt && strlen(fmt) > 0) {
+        va_list va;
+        va_start(va, fmt);
+        VALUE mesg = rb_vsprintf(fmt, va);
+        va_end(va);
+        return rb_exc_new3(exc, mesg);
+    } else {
+        return rb_exc_new2(exc, "");
+    }
+}
+
+VALUE
+extzstd_make_error(ssize_t errcode)
+{
+    if (!ZSTD_isError(errcode)) { return Qnil; }
+
+#define CASE_ERROR(zstderr, extzstderr, ...)                            \
+    case zstderr: return extzstd_make_errorf(extzstderr, __VA_ARGS__);  \
+
+    switch (-errcode) {
+    CASE_ERROR(ZSTD_error_GENERIC, extzstd_eGenericError, NULL);
+    CASE_ERROR(ZSTD_error_prefix_unknown, extzstd_ePrefixUnknownError, NULL);
+    CASE_ERROR(ZSTD_error_frameParameter_unsupported, extzstd_eFrameParameterUnsupportedError, NULL);
+    CASE_ERROR(ZSTD_error_frameParameter_unsupportedBy32bits, extzstd_eFrameParameterUnsupportedBy32bitsError, NULL);
+    CASE_ERROR(ZSTD_error_compressionParameter_unsupported, extzstd_eCompressionParameterUnsupportedError, NULL);
+    CASE_ERROR(ZSTD_error_init_missing, extzstd_eInitMissingError, NULL);
+    CASE_ERROR(ZSTD_error_memory_allocation, extzstd_eMemoryAllocationError, NULL);
+    CASE_ERROR(ZSTD_error_stage_wrong, extzstd_eStageWrongError, NULL);
+    CASE_ERROR(ZSTD_error_dstSize_tooSmall, extzstd_eDstSizeTooSmallError, NULL);
+    CASE_ERROR(ZSTD_error_srcSize_wrong, extzstd_eSrcSizeWrongError, NULL);
+    CASE_ERROR(ZSTD_error_corruption_detected, extzstd_eCorruptionDetectedError, NULL);
+    CASE_ERROR(ZSTD_error_checksum_wrong, extzstd_eChecksumWrongError, NULL);
+    CASE_ERROR(ZSTD_error_tableLog_tooLarge, extzstd_eTableLogTooLargeError, NULL);
+    CASE_ERROR(ZSTD_error_maxSymbolValue_tooLarge, extzstd_eMaxSymbolValueTooLargeError, NULL);
+    CASE_ERROR(ZSTD_error_maxSymbolValue_tooSmall, extzstd_eMaxSymbolValueTooSmallError, NULL);
+    CASE_ERROR(ZSTD_error_dictionary_corrupted, extzstd_eDictionaryCorruptedError, NULL);
+    CASE_ERROR(ZSTD_error_dictionary_wrong, extzstd_eDictionaryWrongError, NULL);
     default:
-        rb_error_arity(argc, 1, 3);
+        return extzstd_make_errorf(extzstd_eError,
+                "unknown zstd error code (%d)", errcode);
     }
+
+#undef CASE_ERROR
+}
+
+static void
+init_error(void)
+{
+    extzstd_mExceptions = rb_define_module_under(extzstd_mZstd, "Exceptions");
+
+    extzstd_eError = rb_define_class_under(extzstd_mZstd, "Error", rb_eRuntimeError);
+    rb_include_module(extzstd_eError, extzstd_mExceptions);
+
+    extzstd_eGenericError = rb_define_class_under(extzstd_mZstd, "GenericError", rb_eRuntimeError);
+    rb_include_module(extzstd_eGenericError, extzstd_mExceptions);
+
+    extzstd_ePrefixUnknownError = rb_define_class_under(extzstd_mZstd, "PrefixUnknownError", rb_eArgError);
+    rb_include_module(extzstd_ePrefixUnknownError, extzstd_mExceptions);
+
+    extzstd_eFrameParameterUnsupportedError = rb_define_class_under(extzstd_mZstd, "FrameParameterUnsupportedError", rb_eRuntimeError);
+    rb_include_module(extzstd_eFrameParameterUnsupportedError, extzstd_mExceptions);
+
+    extzstd_eFrameParameterUnsupportedBy32bitsError = rb_define_class_under(extzstd_mZstd, "FrameParameterUnsupportedBy32bitsImplementationError", rb_eRuntimeError);
+    rb_include_module(extzstd_eFrameParameterUnsupportedBy32bitsError, extzstd_mExceptions);
+
+    extzstd_eCompressionParameterUnsupportedError = rb_define_class_under(extzstd_mZstd, "CompressionParameterUnsupportedError", rb_eRuntimeError);
+    rb_include_module(extzstd_eCompressionParameterUnsupportedError, extzstd_mExceptions);
+
+    extzstd_eInitMissingError = rb_define_class_under(extzstd_mZstd, "InitMissingError", rb_eRuntimeError);
+    rb_include_module(extzstd_eInitMissingError, extzstd_mExceptions);
+
+    extzstd_eMemoryAllocationError = rb_define_class_under(extzstd_mZstd, "MemoryAllocationError", aux_const_dig_str(rb_cObject, "Errno", "ENOMEM"));
+    rb_include_module(extzstd_eMemoryAllocationError, extzstd_mExceptions);
+
+    extzstd_eStageWrongError = rb_define_class_under(extzstd_mZstd, "StageWrongError", rb_eRuntimeError);
+    rb_include_module(extzstd_eStageWrongError, extzstd_mExceptions);
+
+    extzstd_eDstSizeTooSmallError = rb_define_class_under(extzstd_mZstd, "DstSizeTooSmallError", rb_eArgError);
+    rb_include_module(extzstd_eDstSizeTooSmallError, extzstd_mExceptions);
+
+    extzstd_eSrcSizeWrongError = rb_define_class_under(extzstd_mZstd, "SrcSizeWrongError", rb_eArgError);
+    rb_include_module(extzstd_eSrcSizeWrongError, extzstd_mExceptions);
+
+    extzstd_eCorruptionDetectedError = rb_define_class_under(extzstd_mZstd, "CorruptionDetectedError", rb_eRuntimeError);
+    rb_include_module(extzstd_eCorruptionDetectedError, extzstd_mExceptions);
+
+    extzstd_eChecksumWrongError = rb_define_class_under(extzstd_mZstd, "ChecksumWrongError", rb_eRuntimeError);
+    rb_include_module(extzstd_eChecksumWrongError, extzstd_mExceptions);
+
+    extzstd_eTableLogTooLargeError = rb_define_class_under(extzstd_mZstd, "TableLogTooLargeError", rb_eArgError);
+    rb_include_module(extzstd_eTableLogTooLargeError, extzstd_mExceptions);
+
+    extzstd_eMaxSymbolValueTooLargeError = rb_define_class_under(extzstd_mZstd, "MaxSymbolValueTooLargeError", rb_eArgError);
+    rb_include_module(extzstd_eMaxSymbolValueTooLargeError, extzstd_mExceptions);
+
+    extzstd_eMaxSymbolValueTooSmallError = rb_define_class_under(extzstd_mZstd, "MaxSymbolValueTooSmallError", rb_eArgError);
+    rb_include_module(extzstd_eMaxSymbolValueTooSmallError, extzstd_mExceptions);
+
+    extzstd_eDictionaryCorruptedError = rb_define_class_under(extzstd_mZstd, "DictionaryCorruptedError", rb_eRuntimeError);
+    rb_include_module(extzstd_eDictionaryCorruptedError, extzstd_mExceptions);
+
+    extzstd_eDictionaryWrongError = rb_define_class_under(extzstd_mZstd, "DictionaryWrongError", rb_eRuntimeError);
+    rb_include_module(extzstd_eDictionaryWrongError, extzstd_mExceptions);
+}
+
+/*
+ * module Zstd::Constants
+ */
+
+static void
+init_constants(void)
+{
+    VALUE mConstants = rb_define_module_under(extzstd_mZstd, "Constants");
+    rb_include_module(extzstd_mZstd, mConstants);
+
+    rb_define_const(mConstants, "ZSTD_FAST", INT2NUM(ZSTD_fast));
+    rb_define_const(mConstants, "ZSTD_DFAST", INT2NUM(ZSTD_dfast));
+    rb_define_const(mConstants, "ZSTD_GREEDY", INT2NUM(ZSTD_greedy));
+    rb_define_const(mConstants, "ZSTD_LAZY", INT2NUM(ZSTD_lazy));
+    rb_define_const(mConstants, "ZSTD_LAZY2", INT2NUM(ZSTD_lazy2));
+    rb_define_const(mConstants, "ZSTD_BTLAZY2", INT2NUM(ZSTD_btlazy2));
+    rb_define_const(mConstants, "ZSTD_WINDOWLOG_MAX", INT2NUM(ZSTD_WINDOWLOG_MAX));
+    rb_define_const(mConstants, "ZSTD_WINDOWLOG_MIN", INT2NUM(ZSTD_WINDOWLOG_MIN));
+    rb_define_const(mConstants, "ZSTD_CHAINLOG_MAX", INT2NUM(ZSTD_CHAINLOG_MAX));
+    rb_define_const(mConstants, "ZSTD_CHAINLOG_MIN", INT2NUM(ZSTD_CHAINLOG_MIN));
+    rb_define_const(mConstants, "ZSTD_HASHLOG_MAX", INT2NUM(ZSTD_HASHLOG_MAX));
+    rb_define_const(mConstants, "ZSTD_HASHLOG_MIN", INT2NUM(ZSTD_HASHLOG_MIN));
+    rb_define_const(mConstants, "ZSTD_SEARCHLOG_MAX", INT2NUM(ZSTD_SEARCHLOG_MAX));
+    rb_define_const(mConstants, "ZSTD_SEARCHLOG_MIN", INT2NUM(ZSTD_SEARCHLOG_MIN));
+    rb_define_const(mConstants, "ZSTD_SEARCHLENGTH_MAX", INT2NUM(ZSTD_SEARCHLENGTH_MAX));
+    rb_define_const(mConstants, "ZSTD_SEARCHLENGTH_MIN", INT2NUM(ZSTD_SEARCHLENGTH_MIN));
+
+    rb_define_const(mConstants, "FAST", INT2NUM(ZSTD_fast));
+    rb_define_const(mConstants, "DFAST", INT2NUM(ZSTD_dfast));
+    rb_define_const(mConstants, "GREEDY", INT2NUM(ZSTD_greedy));
+    rb_define_const(mConstants, "LAZY", INT2NUM(ZSTD_lazy));
+    rb_define_const(mConstants, "LAZY2", INT2NUM(ZSTD_lazy2));
+    rb_define_const(mConstants, "BTLAZY2", INT2NUM(ZSTD_btlazy2));
+    rb_define_const(mConstants, "WINDOWLOG_MAX", INT2NUM(ZSTD_WINDOWLOG_MAX));
+    rb_define_const(mConstants, "WINDOWLOG_MIN", INT2NUM(ZSTD_WINDOWLOG_MIN));
+    rb_define_const(mConstants, "CHAINLOG_MAX", INT2NUM(ZSTD_CHAINLOG_MAX));
+    rb_define_const(mConstants, "CHAINLOG_MIN", INT2NUM(ZSTD_CHAINLOG_MIN));
+    rb_define_const(mConstants, "HASHLOG_MAX", INT2NUM(ZSTD_HASHLOG_MAX));
+    rb_define_const(mConstants, "HASHLOG_MIN", INT2NUM(ZSTD_HASHLOG_MIN));
+    rb_define_const(mConstants, "SEARCHLOG_MAX", INT2NUM(ZSTD_SEARCHLOG_MAX));
+    rb_define_const(mConstants, "SEARCHLOG_MIN", INT2NUM(ZSTD_SEARCHLOG_MIN));
+    rb_define_const(mConstants, "SEARCHLENGTH_MAX", INT2NUM(ZSTD_SEARCHLENGTH_MAX));
+    rb_define_const(mConstants, "SEARCHLENGTH_MIN", INT2NUM(ZSTD_SEARCHLENGTH_MIN));
+}
+
+/*
+ * class Zstd::EncodeParameter
+ */
+
+VALUE extzstd_cParams;
+
+AUX_IMPLEMENT_CONTEXT(
+        ZSTD_parameters, encparams_type, "extzstd.EncodeParameters",
+        encparams_alloc_dummy, NULL, free, NULL,
+        getencparamsp, getencparams, encparams_p);
+
+ZSTD_parameters *
+extzstd_getencparams(VALUE v)
+{
+    return getencparams(v);
+}
+
+int
+extzstd_encparams_p(VALUE v)
+{
+    return encparams_p(v);
+}
+
+static VALUE
+encparams_alloc(VALUE mod)
+{
+    ZSTD_parameters *p;
+    return TypedData_Make_Struct(mod, ZSTD_parameters, &encparams_type, p);
+}
+
+VALUE
+extzstd_params_alloc(ZSTD_parameters **p)
+{
+    return TypedData_Make_Struct(extzstd_cParams, ZSTD_parameters, &encparams_type, *p);
 }
 
 /*
  * call-seq:
- *  encode(src) -> encoded string
- *  encode(src, size) -> encoded string
- *  encode(src, dest) -> dest with encoded string
- *  encode(src, size, dest) -> dest with encoded string
+ *  initialize(preset_level = 1, srcsize_hint = 0, dictsize = 0, opts = {})
+ *
+ * Initialize struct ZSTD_parameters of C layer.
+ *
+ * [preset_level = 1]
+ * [srcsize_hint = 0]
+ * [opts windowlog: nil]
+ * [opts contentlog: nil]
+ * [opts hashlog: nil]
+ * [opts searchlog: nil]
+ * [opts searchlength: nil]
+ * [opts strategy: nil]
  */
 static VALUE
-zstd_s_encode(int argc, VALUE argv[], VALUE mod)
-{
-    VALUE src, dest;
-    size_t maxsize;
-    zstd_s_encode_args(argc, argv, &src, &dest, &maxsize);
-    const char *srcp;
-    size_t srcsize;
-    RSTRING_GETMEM(src, srcp, srcsize);
-    rb_obj_infect(dest, src);
-    size_t s = aux_ZSTD_compress(RSTRING_PTR(dest), maxsize, srcp, srcsize);
-    if (ZSTD_isError(s)) {
-        rb_raise(eError,
-                "failed ZSTD_compress - %s(%d) in %s:%d:%s",
-                ZSTD_getErrorName(s), (int)s,
-                __FILE__, __LINE__, __func__);
+encparams_init(int argc, VALUE argv[], VALUE v)
+{
+    ZSTD_parameters *p = getencparams(v);
+    uint64_t sizehint;
+    size_t dictsize;
+    int level;
+    VALUE opts = Qnil;
+    int argc0 = argc;
+    if (argc > 0) {
+        if (rb_type_p(argv[argc - 1], RUBY_T_HASH)) {
+            opts = argv[argc - 1];
+            argc --;
+        }
     }
-    if (s > maxsize) {
-        rb_bug("%s:%d:%s: detect buffer overflow in ZSTD_compress - maxsize is %zd, but returned size is %zd",
-                __FILE__, __LINE__, __func__, maxsize, s);
+    if (argc == 0) {
+        level = 0;
+        sizehint = 0;
+        dictsize = 0;
+    } else if (argc == 1) {
+        level = aux_num2int(argv[0], 0);
+        sizehint = 0;
+        dictsize = 0;
+    } else if (argc == 2) {
+        level = aux_num2int(argv[0], 0);
+        sizehint = aux_num2int_u64(argv[1], 0);
+        dictsize = 0;
+    } else if (argc == 3) {
+        level = aux_num2int(argv[0], 0);
+        sizehint = aux_num2int_u64(argv[1], 0);
+        dictsize = aux_num2int_u64(argv[2], 0);
+    } else {
+        rb_raise(rb_eArgError, "wrong number of argument (%d for 0..3 with keywords)", argc0);
     }
-    rb_str_set_len(dest, s);
-    return dest;
-}
 
-static size_t
-aux_ZSTD_decompress_nogvl(va_list *vp)
-{
-    char *dest = va_arg(*vp, char *);
-    size_t destsize = va_arg(*vp, size_t);
-    const char *src = va_arg(*vp, const char *);
-    size_t srcsize = va_arg(*vp, size_t);
-    return ZSTD_decompress(dest, destsize, src, srcsize);
+    *p = ZSTD_getParams(level, sizehint, dictsize);
+
+    if (!NIL_P(opts)) {
+#define SETUP_PARAM(var, opts, key, converter)                          \
+        do {                                                            \
+            VALUE tmp = rb_hash_lookup(opts, ID2SYM(rb_intern(key)));   \
+            if (!NIL_P(tmp)) {                                          \
+                var = converter(tmp);                                   \
+            }                                                           \
+        } while (0)                                                     \
+
+        SETUP_PARAM(p->cParams.windowLog, opts, "windowlog", NUM2UINT);
+        SETUP_PARAM(p->cParams.chainLog, opts, "chainlog", NUM2UINT);
+        SETUP_PARAM(p->cParams.hashLog, opts, "hashlog", NUM2UINT);
+        SETUP_PARAM(p->cParams.searchLog, opts, "searchlog", NUM2UINT);
+        SETUP_PARAM(p->cParams.searchLength, opts, "searchlength", NUM2UINT);
+        SETUP_PARAM(p->cParams.strategy, opts, "strategy", NUM2UINT);
+#undef SETUP_PARAM
+    }
+
+    return v;
 }
 
-static inline size_t
-aux_ZSTD_decompress(char *dest, size_t destsize, const char *src, size_t srcsize)
+static VALUE
+encparams_init_copy(VALUE params, VALUE src)
 {
-    return (size_t)aux_thread_call_without_gvl(
-            (void *(*)(void *))aux_ZSTD_decompress_nogvl, NULL,
-            dest, destsize, src, srcsize);
+    ZSTD_parameters *a = getencparams(params);
+    ZSTD_parameters *b = getencparams(src);
+    rb_check_frozen(params);
+    rb_obj_infect(params, src);
+    memcpy(a, b, sizeof(*a));
+    return params;
 }
 
-static inline void
-zstd_s_decode_args(int argc, VALUE argv[], VALUE *src, VALUE *dest, size_t *maxsize)
+//static VALUE
+//encparams_validate(VALUE v)
+//{
+//    ZSTD_validateParams(getencparams(v));
+//    return v;
+//}
+
+#define IMP_PARAMS(GETTER, SETTER, FIELD)                   \
+    static VALUE                                            \
+    GETTER(VALUE v)                                         \
+    {                                                       \
+        return UINT2NUM(getencparams(v)->cParams.FIELD);    \
+    }                                                       \
+                                                            \
+    static VALUE                                            \
+    SETTER(VALUE v, VALUE n)                                \
+    {                                                       \
+        getencparams(v)->cParams.FIELD = NUM2UINT(n);       \
+        return n;                                           \
+    }                                                       \
+
+//IMP_PARAMS(encparams_srcsize, encparams_set_srcsize, srcSize);
+IMP_PARAMS(encparams_windowlog, encparams_set_windowlog, windowLog);
+IMP_PARAMS(encparams_chainlog, encparams_set_chainlog, chainLog);
+IMP_PARAMS(encparams_hashlog, encparams_set_hashlog, hashLog);
+IMP_PARAMS(encparams_searchlog, encparams_set_searchlog, searchLog);
+IMP_PARAMS(encparams_searchlength, encparams_set_searchlength, searchLength);
+IMP_PARAMS(encparams_strategy, encparams_set_strategy, strategy);
+
+#undef IMP_PARAMS
+
+static VALUE
+encparams_s_get_preset(int argc, VALUE argv[], VALUE mod)
 {
+    int level;
+    uint64_t sizehint;
+    size_t dictsize;
+
     switch (argc) {
+    case 0:
+        level = 0;
+        sizehint = 0;
+        dictsize = 0;
+        break;
+    case 1:
+        level = NUM2INT(argv[0]);
+        sizehint = 0;
+        dictsize = 0;
+        break;
     case 2:
-        *src = argv[0];
-        rb_check_type(*src, RUBY_T_STRING);
-        *maxsize = NUM2SIZET(argv[1]);
-        *dest = rb_str_buf_new(*maxsize);
-        return;
+        level = NUM2INT(argv[0]);
+        sizehint = NUM2ULL(argv[1]);
+        dictsize = 0;
+        break;
     case 3:
-        *src = argv[0];
-        rb_check_type(*src, RUBY_T_STRING);
-        *maxsize = NUM2SIZET(argv[1]);
-        *dest = argv[2];
-        rb_check_type(*dest, RUBY_T_STRING);
-        rb_str_resize(*dest, *maxsize);
-        return;
+        level = NUM2INT(argv[0]);
+        sizehint = NUM2ULL(argv[1]);
+        dictsize = NUM2SIZET(argv[2]);
+        break;
     default:
-        rb_error_arity(argc, 2, 3);
+        rb_error_arity(argc, 0, 3);
     }
+
+    ZSTD_parameters *p;
+    VALUE v = TypedData_Make_Struct(mod, ZSTD_parameters, &encparams_type, p);
+    *p = ZSTD_getParams(level, sizehint, dictsize);
+    return v;
+}
+
+/*
+ * Document-method: Zstd::EncodeParameters#windowlog
+ * Document-method: Zstd::EncodeParameters#windowlog=
+ * Document-method: Zstd::EncodeParameters#chainlog
+ * Document-method: Zstd::EncodeParameters#chainlog=
+ * Document-method: Zstd::EncodeParameters#hashlog
+ * Document-method: Zstd::EncodeParameters#hashlog=
+ * Document-method: Zstd::EncodeParameters#searchlog
+ * Document-method: Zstd::EncodeParameters#searchlog=
+ * Document-method: Zstd::EncodeParameters#searchlength
+ * Document-method: Zstd::EncodeParameters#searchlength=
+ * Document-method: Zstd::EncodeParameters#strategy
+ * Document-method: Zstd::EncodeParameters#strategy=
+ *
+ * Get/Set any field from/to struct ZSTD_parameters of C layer.
+ */
+
+static void
+init_encparams(void)
+{
+    extzstd_cParams = rb_define_class_under(extzstd_mZstd, "EncodeParameters", rb_cObject);
+    rb_define_alloc_func(extzstd_cParams, encparams_alloc);
+    rb_define_method(extzstd_cParams, "initialize", RUBY_METHOD_FUNC(encparams_init), -1);
+    rb_define_method(extzstd_cParams, "initialize_copy", RUBY_METHOD_FUNC(encparams_init_copy), 1);
+    //rb_define_method(extzstd_cParams, "validate", RUBY_METHOD_FUNC(encparams_validate), 0);
+    //rb_define_method(extzstd_cParams, "srcsize", RUBY_METHOD_FUNC(encparams_srcsize), 0);
+    //rb_define_method(extzstd_cParams, "srcsize=", RUBY_METHOD_FUNC(encparams_set_srcsize), 1);
+    rb_define_method(extzstd_cParams, "windowlog", RUBY_METHOD_FUNC(encparams_windowlog), 0);
+    rb_define_method(extzstd_cParams, "windowlog=", RUBY_METHOD_FUNC(encparams_set_windowlog), 1);
+    rb_define_method(extzstd_cParams, "chainlog", RUBY_METHOD_FUNC(encparams_chainlog), 0);
+    rb_define_method(extzstd_cParams, "chainlog=", RUBY_METHOD_FUNC(encparams_set_chainlog), 1);
+    rb_define_method(extzstd_cParams, "hashlog", RUBY_METHOD_FUNC(encparams_hashlog), 0);
+    rb_define_method(extzstd_cParams, "hashlog=", RUBY_METHOD_FUNC(encparams_set_hashlog), 1);
+    rb_define_method(extzstd_cParams, "searchlog", RUBY_METHOD_FUNC(encparams_searchlog), 0);
+    rb_define_method(extzstd_cParams, "searchlog=", RUBY_METHOD_FUNC(encparams_set_searchlog), 1);
+    rb_define_method(extzstd_cParams, "searchlength", RUBY_METHOD_FUNC(encparams_searchlength), 0);
+    rb_define_method(extzstd_cParams, "searchlength=", RUBY_METHOD_FUNC(encparams_set_searchlength), 1);
+    rb_define_method(extzstd_cParams, "strategy", RUBY_METHOD_FUNC(encparams_strategy), 0);
+    rb_define_method(extzstd_cParams, "strategy=", RUBY_METHOD_FUNC(encparams_set_strategy), 1);
+
+    rb_define_singleton_method(extzstd_cParams, "preset", RUBY_METHOD_FUNC(encparams_s_get_preset), -1);
+    rb_define_alias(rb_singleton_class(extzstd_cParams), "[]", "preset");
 }
 
+
+/*
+ * zstd dictionary utilities
+ */
+
 /*
  * call-seq:
- *  decode(src, size) -> decoded string
- *  decode(src, size, dest) -> dest with decoded string
+ *  dict_train_from_buffer(src, dict_capacity) -> dictionary'd string
  */
 static VALUE
-zstd_s_decode(int argc, VALUE argv[], VALUE mod)
-{
-    VALUE src, dest;
-    size_t maxsize;
-    zstd_s_decode_args(argc, argv, &src, &dest, &maxsize);
-    const char *srcp;
-    size_t srcsize;
-    RSTRING_GETMEM(src, srcp, srcsize);
-    rb_obj_infect(dest, src);
-    size_t s = aux_ZSTD_decompress(RSTRING_PTR(dest), maxsize, srcp, srcsize);
-    if (ZSTD_isError(s)) {
-        rb_raise(eError,
-                "failed ZSTD_decompress - %s(%d) in %s:%d:%s",
-                ZSTD_getErrorName(s), (int)s,
-                __FILE__, __LINE__, __func__);
-    }
-    if (s > maxsize) {
-        rb_bug("%s:%d:%s: detect buffer overflow in ZSTD_compress - maxsize is %zd, but returned size is %zd",
-                __FILE__, __LINE__, __func__, maxsize, s);
-    }
-    rb_str_set_len(dest, s);
-    return dest;
+ext_s_dict_train_from_buffer(VALUE mod, VALUE src, VALUE dict_capacity)
+{
+    rb_check_type(src, RUBY_T_STRING);
+    size_t capa = NUM2SIZET(dict_capacity);
+    VALUE dict = rb_str_buf_new(capa);
+    size_t srcsize = RSTRING_LEN(src);
+    size_t s = ZDICT_trainFromBuffer(RSTRING_PTR(dict), capa, RSTRING_PTR(src), &srcsize, 1);
+    extzstd_check_error(s);
+    rb_str_set_len(dict, s);
+    return dict;
 }
 
+/*
+ * call-seq:
+ *  dict_add_entropy_tables_from_buffer(dict, dict_capacity, sample) -> dict
+ */
 static VALUE
-libver_s_to_s(VALUE ver)
+ext_s_dict_add_entropy_tables_from_buffer(VALUE mod, VALUE dict, VALUE dict_capacity, VALUE sample)
 {
-    static VALUE str;
-    if (!str) {
-        str = rb_sprintf("%d.%d.%d",
-                ZSTD_VERSION_MAJOR,
-                ZSTD_VERSION_MINOR,
-                ZSTD_VERSION_RELEASE);
-    }
-    return str;
+    /*
+     * size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+     *                                         const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
+     */
+
+    rb_check_type(dict, RUBY_T_STRING);
+    rb_check_type(sample, RUBY_T_STRING);
+    size_t capa = NUM2SIZET(dict_capacity);
+    aux_str_modify_expand(dict, capa);
+    size_t samplesize = RSTRING_LEN(sample);
+    size_t s = ZDICT_addEntropyTablesFromBuffer(RSTRING_PTR(dict), RSTRING_LEN(dict), capa, RSTRING_PTR(dict), &samplesize, 1);
+    extzstd_check_error(s);
+    rb_str_set_len(dict, s);
+    return dict;
 }
 
-void
-Init_extzstd(void)
+static void
+init_dictionary(void)
 {
-    mZstd = rb_define_module("Zstd");
-    rb_define_singleton_method(mZstd, "encode", RUBY_METHOD_FUNC(zstd_s_encode), -1);
-    rb_define_singleton_method(mZstd, "decode", RUBY_METHOD_FUNC(zstd_s_decode), -1);
+    rb_define_singleton_method(extzstd_mZstd, "dict_train_from_buffer", ext_s_dict_train_from_buffer, 2);
+    rb_define_singleton_method(extzstd_mZstd, "dict_add_entropy_tables_from_buffer", ext_s_dict_add_entropy_tables_from_buffer, 3);
+}
 
-    VALUE libver = rb_ary_new3(3,
-            INT2FIX(ZSTD_VERSION_MAJOR),
-            INT2FIX(ZSTD_VERSION_MINOR),
-            INT2FIX(ZSTD_VERSION_RELEASE));
-    rb_define_singleton_method(libver, "to_s", RUBY_METHOD_FUNC(libver_s_to_s), 0);
-    rb_obj_freeze(libver);
-    rb_define_const(mZstd, "LIBRARY_VERSION", libver);
 
-    eError = rb_define_class_under(mZstd, "Error", rb_eRuntimeError);
+/*
+ * library initializer
+ */
+
+void
+Init_extzstd(void)
+{
+    extzstd_mZstd = rb_define_module("Zstd");
 
-    init_extzstd_stream();
+    init_libver();
+    init_error();
+    init_constants();
+    init_encparams();
+    init_dictionary();
+    extzstd_init_buffered();
 }