multi_compress 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6af5cef6e9b15c5b0e11a148b0543377c0ad332e710083950bca7cb3d49e70e
-  data.tar.gz: 6722c74ed13eabcdb3a4a7ca81cd12b9d3f286f11d00e73de759861db198f2a2
+  metadata.gz: 51bd49ce7dbb59d7428cec080e9f2e1dc3b83412848b8e30e37eec3dc7f4fbf4
+  data.tar.gz: 34bd66cd9e255a5e13bb48f91afb584e2c3f3caefed71bd5c8367f61d6199e7d
 SHA512:
-  metadata.gz: bca66c39d081538682abd1a45862f4d141605f028b3f233ec886d21f3b38c84000956275b73303c00fda5755f37796c3bcd8bb533db32a0f12b272107ed298c1
-  data.tar.gz: 209cebbbbc4be03bd48eea3d59ec1254502db4a8a5e9109b5f03425cc2f05d474ab7f250d2a2ecfaba36f1c6ac9686b159cf0433bb9c50bd84aecffe62ab365e
+  metadata.gz: 3c2139458b9ec45769988f3efb3a3165c96801436ca31c7086f6f56a1a0549095d4b493fe239ea1607932b4209315eab076a2ed5fb68ac2249af5175382ec898
+  data.tar.gz: 78290aed6265c4915f7eb850646fbdc0e6000a615312a4714c8dbf1aac7d8817e88bcef06b92507d718a2f774601b183c521f05f7c12758bc85d04407afb51f9

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.3.4]
+
+### Changed
+- Improved one-shot Zstd performance by reusing per-thread CCtx/DCtx.
+- On deterministic arm64-darwin benchmarks, total zstd roundtrip improved by ~11–19% on ~10KB payloads,
+  ~1–4% on medium payloads, and ~2–6% on log-like large payloads, with identical compressed sizes.
+
 ## [0.3.3]
 
 ### Changed

data/ext/multi_compress/extconf.rb

@@ -6,6 +6,8 @@ USE_SYSTEM = arg_config("--use-system-libraries") ||
              ENV["COMPRESS_USE_SYSTEM_LIBRARIES"]
 FORCE_VENDORED = arg_config("--force-vendored") ||
                  ENV["COMPRESS_FORCE_VENDORED"]
+DISABLE_ZSTD_ASM = arg_config("--disable-zstd-asm") ||
+                   ENV["MULTI_COMPRESS_DISABLE_ZSTD_ASM"] == "1"
 
 ZSTD_SUBDIRS   = %w[lib/common lib/compress lib/decompress lib/dictBuilder].freeze
 BROTLI_SUBDIRS = %w[c/common c/enc c/dec].freeze
@@ -53,6 +55,15 @@ def find_compress_c_dir
             &.then { |path| File.expand_path(path) } || __dir__
 end
 
+def zstd_asm_supported?
+  case RUBY_PLATFORM
+  when /x86_64|amd64/
+    !RUBY_PLATFORM.include?("mswin") && !RUBY_PLATFORM.include?("mingw")
+  else
+    false
+  end
+end
+
 def configure_system_libraries
   puts "Building with SYSTEM libraries"
 
@@ -98,7 +109,10 @@ def configure_vendored_libraries(vendor_dir)
   puts "  #{all_vendor_srcs.length} vendored C files"
 
   add_include_dirs(zstd_dir, lz4_dir, brotli_dir)
-  $CPPFLAGS += " -DZSTD_DISABLE_ASM"
+  if DISABLE_ZSTD_ASM
+    $CPPFLAGS += " -DZSTD_DISABLE_ASM"
+    puts "  ZSTD ASM Huffman decoder disabled (--disable-zstd-asm or MULTI_COMPRESS_DISABLE_ZSTD_ASM=1)"
+  end
 
   vpath_dirs = build_vpath_dirs(zstd_dir, lz4_dir, brotli_dir)
 
@@ -106,9 +120,14 @@ def configure_vendored_libraries(vendor_dir)
 
   compress_c_dir = find_compress_c_dir
 
-  $srcs  = ["multi_compress.c"] + all_vendor_srcs.map { |s| File.basename(s) }
+  c_srcs   = all_vendor_srcs.reject { |s| s.end_with?(".S") }
+  asm_srcs = all_vendor_srcs.select { |s| s.end_with?(".S") }
+
+  $srcs  = ["multi_compress.c"] + c_srcs.map { |s| File.basename(s) }
   $VPATH = [compress_c_dir] + vpath_dirs
 
+  $multi_compress_asm_srcs = asm_srcs
+
   $warnflags = ""
 
   vpath_dirs
@@ -117,6 +136,11 @@ end
 def collect_vendor_sources(zstd_dir, lz4_dir, brotli_dir)
   zstd_srcs = ZSTD_SUBDIRS.flat_map { |d| Dir[File.join(zstd_dir, d, "*.c")] }
 
+  unless DISABLE_ZSTD_ASM
+    asm = File.join(zstd_dir, "lib", "decompress", "huf_decompress_amd64.S")
+    zstd_srcs << asm if File.exist?(asm) && zstd_asm_supported?
+  end
+
   lz4_srcs = LZ4_SOURCES.filter_map do |f|
     path = File.join(lz4_dir, "lib", f)
     path if File.exist?(path)
@@ -178,6 +202,36 @@ def patch_makefile_vpath!(vpath_dirs)
   puts "  Patched Makefile with #{vpath_dirs.length} VPATH entries"
 end
 
+def patch_makefile_asm!(asm_srcs)
+  return if asm_srcs.nil? || asm_srcs.empty?
+
+  makefile = File.read("Makefile")
+  return if makefile.include?("# vendored asm")
+
+  asm_dirs = asm_srcs.map { |s| File.dirname(s) }.uniq
+  vpath_lines = asm_dirs.map { |d| "vpath %.S #{d}" }.join("\n")
+
+  asm_objs = asm_srcs.map { |s| File.basename(s, ".S") + ".o" }
+  obj_append = asm_objs.join(" ")
+
+  unless makefile.sub!(/^(OBJS\s*=\s*[^\n]+?)(\s*)$/) { "#{Regexp.last_match(1)} #{obj_append}#{Regexp.last_match(2)}" }
+    makefile << "\nOBJS = #{obj_append}\n"
+  end
+
+  pattern_rule = <<~MAKE
+    # vendored asm
+    #{vpath_lines}
+    %.o: %.S
+    \t$(ECHO) compiling $(<)
+    \t$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $<
+  MAKE
+
+  makefile << "\n#{pattern_rule}\n"
+
+  File.write("Makefile", makefile)
+  puts "  Patched Makefile with #{asm_srcs.length} ASM source(s): #{asm_objs.join(", ")}"
+end
+
 # --- Main ---
 
 VENDOR_DIR = find_vendor_dir
@@ -211,3 +265,4 @@ have_library("pthread") unless RUBY_PLATFORM.include?("darwin")
 create_makefile("multi_compress/multi_compress")
 
 patch_makefile_vpath!(vpath_dirs) if VENDORED && !USE_SYSTEM && vpath_dirs
+patch_makefile_asm!($multi_compress_asm_srcs) if VENDORED && !USE_SYSTEM && $multi_compress_asm_srcs

data/ext/multi_compress/multi_compress.c

@@ -95,6 +95,97 @@ typedef enum { LZ4_FORMAT_BLOCK = 0, LZ4_FORMAT_FRAME = 1 } lz4_format_t;
 
 #define MC_NUM_ALGOS 3
 
+static pthread_once_t zstd_tls_once = PTHREAD_ONCE_INIT;
+static pthread_key_t zstd_cctx_key;
+static pthread_key_t zstd_dctx_key;
+
+static void zstd_tls_free_cctx(void *ptr) {
+    if (ptr)
+        ZSTD_freeCCtx((ZSTD_CCtx *)ptr);
+}
+
+static void zstd_tls_free_dctx(void *ptr) {
+    if (ptr)
+        ZSTD_freeDCtx((ZSTD_DCtx *)ptr);
+}
+
+static void zstd_tls_init(void) {
+    if (pthread_key_create(&zstd_cctx_key, zstd_tls_free_cctx) != 0)
+        abort();
+    if (pthread_key_create(&zstd_dctx_key, zstd_tls_free_dctx) != 0)
+        abort();
+}
+
+static ZSTD_CCtx *zstd_tls_get_cctx(void) {
+    pthread_once(&zstd_tls_once, zstd_tls_init);
+
+    ZSTD_CCtx *cctx = (ZSTD_CCtx *)pthread_getspecific(zstd_cctx_key);
+    if (cctx)
+        return cctx;
+
+    cctx = ZSTD_createCCtx();
+    if (!cctx)
+        return NULL;
+
+    if (pthread_setspecific(zstd_cctx_key, cctx) != 0) {
+        ZSTD_freeCCtx(cctx);
+        return NULL;
+    }
+
+    return cctx;
+}
+
+static ZSTD_DCtx *zstd_tls_get_dctx(void) {
+    pthread_once(&zstd_tls_once, zstd_tls_init);
+
+    ZSTD_DCtx *dctx = (ZSTD_DCtx *)pthread_getspecific(zstd_dctx_key);
+    if (dctx)
+        return dctx;
+
+    dctx = ZSTD_createDCtx();
+    if (!dctx)
+        return NULL;
+
+    if (pthread_setspecific(zstd_dctx_key, dctx) != 0) {
+        ZSTD_freeDCtx(dctx);
+        return NULL;
+    }
+
+    return dctx;
+}
+
+static size_t zstd_compress_cached(char *dst, size_t dst_cap, const char *src, size_t src_len,
+                                   int level, ZSTD_CDict *cdict, int *ctx_error) {
+    *ctx_error = 0;
+
+    ZSTD_CCtx *cctx = zstd_tls_get_cctx();
+    if (!cctx) {
+        *ctx_error = 1;
+        return 0;
+    }
+
+    if (cdict)
+        return ZSTD_compress_usingCDict(cctx, dst, dst_cap, src, src_len, cdict);
+
+    return ZSTD_compressCCtx(cctx, dst, dst_cap, src, src_len, level);
+}
+
+static size_t zstd_decompress_cached(void *dst, size_t dst_cap, const void *src, size_t src_len,
+                                     ZSTD_DDict *ddict, int *ctx_error) {
+    *ctx_error = 0;
+
+    ZSTD_DCtx *dctx = zstd_tls_get_dctx();
+    if (!dctx) {
+        *ctx_error = 1;
+        return 0;
+    }
+
+    if (ddict)
+        return ZSTD_decompress_usingDDict(dctx, dst, dst_cap, src, src_len, ddict);
+
+    return ZSTD_decompressDCtx(dctx, dst, dst_cap, src, src_len);
+}
+
 _Static_assert(ALGO_BROTLI == MC_NUM_ALGOS - 1,
                "compress_algo_t must be contiguous [0..MC_NUM_ALGOS-1]");
 
@@ -439,8 +530,6 @@ static inline VALUE rb_binary_str_buf_new(long capa) {
 static inline VALUE rb_binary_str_buf_reserve(long capa) {
     VALUE str = rb_str_buf_new(capa);
     rb_enc_associate(str, binary_encoding);
-    if (capa > 0)
-        rb_str_modify_expand(str, capa + 1);
     return str;
 }
 
@@ -689,13 +778,19 @@ static inline size_t fiber_maybe_yield(size_t bytes_since_yield, size_t just_pro
     return bytes_since_yield;
 }
 
-#define DICT_CDICT_CACHE_SIZE 22
-_Static_assert(DICT_CDICT_CACHE_SIZE > 0, "CDict cache needs at least one slot");
+#define DICT_ZSTD_MIN_LEVEL   1
+#define DICT_ZSTD_MAX_LEVEL   22
+#define DICT_CDICT_CACHE_SIZE (DICT_ZSTD_MAX_LEVEL + 1)
+_Static_assert(DICT_CDICT_CACHE_SIZE > DICT_ZSTD_MAX_LEVEL,
+               "CDict cache needs one slot for every accepted zstd level");
 
-typedef struct {
-    int level;
-    ZSTD_CDict *cdict;
-} cdict_cache_entry_t;
+#if defined(__GNUC__) || defined(__clang__)
+#define MC_HAS_ATOMIC_PTR             1
+#define MC_ATOMIC_LOAD_PTR(ptr)       __atomic_load_n((ptr), __ATOMIC_ACQUIRE)
+#define MC_ATOMIC_STORE_PTR(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELEASE)
+#else
+#define MC_HAS_ATOMIC_PTR 0
+#endif
 
 struct dictionary_s {
     compress_algo_t algo;
@@ -703,8 +798,7 @@ struct dictionary_s {
     size_t size;
     pthread_mutex_t cache_mutex;
 
-    cdict_cache_entry_t cdict_cache[DICT_CDICT_CACHE_SIZE];
-    int cdict_cache_count;
+    ZSTD_CDict *cdict_cache[DICT_CDICT_CACHE_SIZE];
 
     ZSTD_DDict *ddict;
 };
@@ -713,9 +807,9 @@ static void dict_free(void *ptr) {
     dictionary_t *dict = (dictionary_t *)ptr;
     if (!dict)
         return;
-    for (int i = 0; i < dict->cdict_cache_count; i++) {
-        if (dict->cdict_cache[i].cdict)
-            ZSTD_freeCDict(dict->cdict_cache[i].cdict);
+    for (int i = DICT_ZSTD_MIN_LEVEL; i <= DICT_ZSTD_MAX_LEVEL; i++) {
+        if (dict->cdict_cache[i])
+            ZSTD_freeCDict(dict->cdict_cache[i]);
     }
     if (dict->ddict)
         ZSTD_freeDDict(dict->ddict);
@@ -732,9 +826,9 @@ static size_t dict_memsize(const void *ptr) {
 
     size_t total = sizeof(dictionary_t) + d->size;
     if (d->algo == ALGO_ZSTD) {
-        for (int i = 0; i < d->cdict_cache_count; i++) {
-            if (d->cdict_cache[i].cdict)
-                total += ZSTD_sizeof_CDict(d->cdict_cache[i].cdict);
+        for (int i = DICT_ZSTD_MIN_LEVEL; i <= DICT_ZSTD_MAX_LEVEL; i++) {
+            if (d->cdict_cache[i])
+                total += ZSTD_sizeof_CDict(d->cdict_cache[i]);
         }
         if (d->ddict)
             total += ZSTD_sizeof_DDict(d->ddict);
@@ -756,17 +850,18 @@ static VALUE dict_alloc(VALUE klass) {
 }
 
 static ZSTD_CDict *dict_get_cdict(dictionary_t *dict, int level) {
-    ZSTD_CDict *existing = NULL;
+    if (MC_UNLIKELY(level < DICT_ZSTD_MIN_LEVEL || level > DICT_ZSTD_MAX_LEVEL))
+        rb_raise(eLevelError, "zstd level must be %d..%d, got %d", DICT_ZSTD_MIN_LEVEL,
+                 DICT_ZSTD_MAX_LEVEL, level);
 
+    ZSTD_CDict *existing;
+#if MC_HAS_ATOMIC_PTR
+    existing = MC_ATOMIC_LOAD_PTR(&dict->cdict_cache[level]);
+#else
     pthread_mutex_lock(&dict->cache_mutex);
-    for (int i = 0; i < dict->cdict_cache_count; i++) {
-        if (dict->cdict_cache[i].level == level) {
-            existing = dict->cdict_cache[i].cdict;
-            break;
-        }
-    }
+    existing = dict->cdict_cache[level];
     pthread_mutex_unlock(&dict->cache_mutex);
-
+#endif
     if (existing)
         return existing;
 
@@ -775,34 +870,31 @@ static ZSTD_CDict *dict_get_cdict(dictionary_t *dict, int level) {
         return NULL;
 
     pthread_mutex_lock(&dict->cache_mutex);
-    for (int i = 0; i < dict->cdict_cache_count; i++) {
-        if (dict->cdict_cache[i].level == level) {
-            existing = dict->cdict_cache[i].cdict;
-            pthread_mutex_unlock(&dict->cache_mutex);
-            ZSTD_freeCDict(cdict);
-            return existing;
-        }
-    }
-
-    if (dict->cdict_cache_count >= DICT_CDICT_CACHE_SIZE) {
+    existing = dict->cdict_cache[level];
+    if (!existing) {
+#if MC_HAS_ATOMIC_PTR
+        MC_ATOMIC_STORE_PTR(&dict->cdict_cache[level], cdict);
+#else
+        dict->cdict_cache[level] = cdict;
+#endif
         pthread_mutex_unlock(&dict->cache_mutex);
-        ZSTD_freeCDict(cdict);
-        rb_raise(eError, "zstd dictionary cdict cache exhausted");
+        return cdict;
     }
 
-    dict->cdict_cache[dict->cdict_cache_count].level = level;
-    dict->cdict_cache[dict->cdict_cache_count].cdict = cdict;
-    dict->cdict_cache_count++;
     pthread_mutex_unlock(&dict->cache_mutex);
-    return cdict;
+    ZSTD_freeCDict(cdict);
+    return existing;
 }
 
 static ZSTD_DDict *dict_get_ddict(dictionary_t *dict) {
     ZSTD_DDict *existing;
-
+#if MC_HAS_ATOMIC_PTR
+    existing = MC_ATOMIC_LOAD_PTR(&dict->ddict);
+#else
     pthread_mutex_lock(&dict->cache_mutex);
     existing = dict->ddict;
     pthread_mutex_unlock(&dict->cache_mutex);
+#endif
     if (existing)
         return existing;
 
@@ -811,12 +903,17 @@ static ZSTD_DDict *dict_get_ddict(dictionary_t *dict) {
         return NULL;
 
     pthread_mutex_lock(&dict->cache_mutex);
-    if (!dict->ddict) {
+    existing = dict->ddict;
+    if (!existing) {
+#if MC_HAS_ATOMIC_PTR
+        MC_ATOMIC_STORE_PTR(&dict->ddict, created);
+#else
         dict->ddict = created;
+#endif
         pthread_mutex_unlock(&dict->cache_mutex);
         return created;
     }
-    existing = dict->ddict;
+
     pthread_mutex_unlock(&dict->cache_mutex);
     ZSTD_freeDDict(created);
     return existing;
@@ -835,19 +932,8 @@ typedef struct {
 
 static void *zstd_compress_nogvl(void *arg) {
     zstd_compress_args_t *a = (zstd_compress_args_t *)arg;
-    if (a->cdict) {
-        ZSTD_CCtx *cctx = ZSTD_createCCtx();
-        if (!cctx) {
-            a->error = 1;
-            return NULL;
-        }
-        a->result =
-            ZSTD_compress_usingCDict(cctx, a->dst, a->dst_cap, a->src, a->src_len, a->cdict);
-        ZSTD_freeCCtx(cctx);
-    } else {
-        a->result = ZSTD_compress(a->dst, a->dst_cap, a->src, a->src_len, a->level);
-    }
-    a->error = 0;
+    a->result =
+        zstd_compress_cached(a->dst, a->dst_cap, a->src, a->src_len, a->level, a->cdict, &a->error);
     return NULL;
 }
 
@@ -863,19 +949,7 @@ typedef struct {
 
 static void *zstd_decompress_nogvl(void *arg) {
     zstd_decompress_args_t *a = (zstd_decompress_args_t *)arg;
-    if (a->ddict) {
-        ZSTD_DCtx *dctx = ZSTD_createDCtx();
-        if (!dctx) {
-            a->error = 1;
-            return NULL;
-        }
-        a->result =
-            ZSTD_decompress_usingDDict(dctx, a->dst, a->dst_cap, a->src, a->src_len, a->ddict);
-        ZSTD_freeDCtx(dctx);
-    } else {
-        a->result = ZSTD_decompress(a->dst, a->dst_cap, a->src, a->src_len);
-    }
-    a->error = 0;
+    a->result = zstd_decompress_cached(a->dst, a->dst_cap, a->src, a->src_len, a->ddict, &a->error);
     return NULL;
 }
 
@@ -1168,18 +1242,8 @@ static void *brotli_decompress_stream_fiber_nogvl(void *arg) {
 
 static void *zstd_fiber_compress_nogvl(void *arg) {
     zstd_fiber_compress_t *a = (zstd_fiber_compress_t *)arg;
-    if (a->cdict) {
-        ZSTD_CCtx *cctx = ZSTD_createCCtx();
-        if (!cctx) {
-            a->error = 1;
-            return NULL;
-        }
-        a->result =
-            ZSTD_compress_usingCDict(cctx, a->dst, a->dst_cap, a->src, a->src_len, a->cdict);
-        ZSTD_freeCCtx(cctx);
-    } else {
-        a->result = ZSTD_compress(a->dst, a->dst_cap, a->src, a->src_len, a->level);
-    }
+    a->result =
+        zstd_compress_cached(a->dst, a->dst_cap, a->src, a->src_len, a->level, a->cdict, &a->error);
     return NULL;
 }
 
@@ -1226,16 +1290,11 @@ static VALUE compress_compress(int argc, VALUE *argv, VALUE self) {
 
         if (slen < policy->gvl_unlock_threshold) {
             VALUE dst = rb_binary_str_buf_reserve(bound);
-            size_t csize;
-            if (cdict) {
-                ZSTD_CCtx *cctx = ZSTD_createCCtx();
-                if (!cctx)
-                    rb_raise(eMemError, "zstd: failed to create context");
-                csize = ZSTD_compress_usingCDict(cctx, RSTRING_PTR(dst), bound, src, slen, cdict);
-                ZSTD_freeCCtx(cctx);
-            } else {
-                csize = ZSTD_compress(RSTRING_PTR(dst), bound, src, slen, level);
-            }
+            int ctx_error = 0;
+            size_t csize =
+                zstd_compress_cached(RSTRING_PTR(dst), bound, src, slen, level, cdict, &ctx_error);
+            if (ctx_error)
+                rb_raise(eMemError, "zstd: failed to create context");
             if (ZSTD_isError(csize))
                 rb_raise(eError, "zstd compress: %s", ZSTD_getErrorName(csize));
             rb_str_set_len(dst, (long)csize);
@@ -1416,8 +1475,18 @@ static VALUE compress_compress(int argc, VALUE *argv, VALUE self) {
                 rb_raise(eMemError, "brotli: failed to prepare dictionary");
             }
 
-            if (!BrotliEncoderSetParameter(enc, BROTLI_PARAM_QUALITY, level) ||
-                !BrotliEncoderAttachPreparedDictionary(enc, pd)) {
+            if (!BrotliEncoderSetParameter(enc, BROTLI_PARAM_QUALITY, level)) {
+                BrotliEncoderDestroyPreparedDictionary(pd);
+                BrotliEncoderDestroyInstance(enc);
+                rb_raise(eError, "brotli: failed to set quality parameter");
+            }
+            if (!BrotliEncoderSetParameter(enc, BROTLI_PARAM_SIZE_HINT,
+                                           slen > UINT32_MAX ? UINT32_MAX : (uint32_t)slen)) {
+                BrotliEncoderDestroyPreparedDictionary(pd);
+                BrotliEncoderDestroyInstance(enc);
+                rb_raise(eError, "brotli: failed to set size hint parameter");
+            }
+            if (!BrotliEncoderAttachPreparedDictionary(enc, pd)) {
                 BrotliEncoderDestroyPreparedDictionary(pd);
                 BrotliEncoderDestroyInstance(enc);
                 rb_raise(eError, "brotli: failed to attach dictionary");
@@ -1576,20 +1645,19 @@ static VALUE compress_decompress(int argc, VALUE *argv, VALUE self) {
             } else {
                 VALUE dst = rb_binary_str_buf_reserve((size_t)frame_size);
 
+                ZSTD_DDict *ddict = NULL;
                 if (dict) {
-                    ZSTD_DDict *ddict = dict_get_ddict(dict);
+                    ddict = dict_get_ddict(dict);
                     if (!ddict)
                         rb_raise(eMemError, "zstd: failed to create ddict");
-                    ZSTD_DCtx *dctx = ZSTD_createDCtx();
-                    if (!dctx)
-                        rb_raise(eMemError, "zstd: failed to create dctx");
-                    dsize = ZSTD_decompress_usingDDict(dctx, RSTRING_PTR(dst), (size_t)frame_size,
-                                                       src, slen, ddict);
-                    ZSTD_freeDCtx(dctx);
-                } else {
-                    dsize = ZSTD_decompress(RSTRING_PTR(dst), (size_t)frame_size, src, slen);
                 }
 
+                int ctx_error = 0;
+                dsize = zstd_decompress_cached(RSTRING_PTR(dst), (size_t)frame_size, src, slen,
+                                               ddict, &ctx_error);
+                if (ctx_error)
+                    rb_raise(eMemError, "zstd: failed to create dctx");
+
                 if (ZSTD_isError(dsize))
                     rb_raise(eDataError, "zstd decompress: %s", ZSTD_getErrorName(dsize));
                 enforce_output_and_ratio_limits(dsize, slen, limits.max_output_size,
@@ -1601,16 +1669,21 @@ static VALUE compress_decompress(int argc, VALUE *argv, VALUE self) {
             }
         }
 
-        ZSTD_DCtx *dctx = ZSTD_createDCtx();
+        ZSTD_DCtx *dctx = zstd_tls_get_dctx();
         if (!dctx)
             rb_raise(eMemError, "zstd: failed to create dctx");
 
+        {
+            size_t r = ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+            if (ZSTD_isError(r))
+                rb_raise(eError, "zstd dctx reset: %s", ZSTD_getErrorName(r));
+        }
+
         if (dict) {
             ZSTD_DDict *ddict = dict_get_ddict(dict);
             if (ddict) {
                 size_t r = ZSTD_DCtx_refDDict(dctx, ddict);
                 if (ZSTD_isError(r)) {
-                    ZSTD_freeDCtx(dctx);
                     rb_raise(eError, "zstd dict ref: %s", ZSTD_getErrorName(r));
                 }
             }
@@ -1629,7 +1702,6 @@ static VALUE compress_decompress(int argc, VALUE *argv, VALUE self) {
         while (input.pos < input.size) {
             if (total_out >= alloc_size) {
                 if (alloc_size >= limits.max_output_size) {
-                    ZSTD_freeDCtx(dctx);
                     rb_raise(eDataError, "decompressed output exceeds limit (%zu bytes)",
                              limits.max_output_size);
                 }
@@ -1642,7 +1714,6 @@ static VALUE compress_decompress(int argc, VALUE *argv, VALUE self) {
 
             size_t remaining_budget = limits.max_output_size - total_out;
             if (remaining_budget == 0) {
-                ZSTD_freeDCtx(dctx);
                 rb_raise(eDataError, "decompressed output exceeds limit (%zu bytes)",
                          limits.max_output_size);
             }
@@ -1654,7 +1725,6 @@ static VALUE compress_decompress(int argc, VALUE *argv, VALUE self) {
             ZSTD_outBuffer output = {RSTRING_PTR(dst) + total_out, out_cap, 0};
             size_t ret = ZSTD_decompressStream(dctx, &output, &input);
             if (ZSTD_isError(ret)) {
-                ZSTD_freeDCtx(dctx);
                 rb_raise(eDataError, "zstd decompress: %s", ZSTD_getErrorName(ret));
             }
             total_out = checked_add_size(total_out, output.pos,
@@ -1665,7 +1735,6 @@ static VALUE compress_decompress(int argc, VALUE *argv, VALUE self) {
                 break;
         }
 
-        ZSTD_freeDCtx(dctx);
         rb_str_set_len(dst, total_out);
         RB_GC_GUARD(data);
         RB_GC_GUARD(dict_val);
@@ -1875,7 +1944,7 @@ static void crc32_init_tables(void) {
     for (uint32_t i = 0; i < 256; i++) {
         uint32_t crc = i;
         for (int j = 0; j < 8; j++) {
-            crc = (crc >> 1) ^ (0xEDB88320 & (-(int32_t)(crc & 1)));
+            crc = (crc >> 1) ^ (0xEDB88320u & (0u - (crc & 1u)));
         }
         crc32_tables[0][i] = crc;
     }
@@ -2189,6 +2258,10 @@ static VALUE lz4_compress_ring_block(deflater_t *d) {
 
     write_le_u32((uint8_t *)out, (uint32_t)src_size);
 
+    /* Keep blocks independently decodable. Switching to LZ4_*_continue would
+     * require a coordinated format/decoder change that preserves dictionaries
+     * across blocks.
+     */
     int csize;
     if (d->level > 1) {
         csize = LZ4_compress_HC(block_start, out + 8, src_size, bound, d->level);
@@ -2979,13 +3052,24 @@ static VALUE inflater_write(VALUE self, VALUE chunk) {
     }
     case ALGO_LZ4: {
         size_t data_len = inf->lz4_buf.len - inf->lz4_buf.offset;
-        size_t needed = data_len + slen;
+        size_t needed =
+            checked_add_size(data_len, slen, "lz4 stream input buffer exceeds representable size");
 
-        if (inf->lz4_buf.offset > 0 && needed > inf->lz4_buf.cap) {
-            if (data_len > 0)
-                memmove(inf->lz4_buf.buf, inf->lz4_buf.buf + inf->lz4_buf.offset, data_len);
-            inf->lz4_buf.offset = 0;
-            inf->lz4_buf.len = data_len;
+        if (needed > inf->lz4_buf.cap) {
+            size_t new_cap = needed > SIZE_MAX / 2 ? needed : needed * 2;
+            if (inf->lz4_buf.offset > 0) {
+                char *new_buf = ALLOC_N(char, new_cap);
+                if (data_len > 0)
+                    memcpy(new_buf, inf->lz4_buf.buf + inf->lz4_buf.offset, data_len);
+                xfree(inf->lz4_buf.buf);
+                inf->lz4_buf.buf = new_buf;
+                inf->lz4_buf.offset = 0;
+                inf->lz4_buf.len = data_len;
+                inf->lz4_buf.cap = new_cap;
+            } else {
+                REALLOC_N(inf->lz4_buf.buf, char, new_cap);
+                inf->lz4_buf.cap = new_cap;
+            }
         } else if (inf->lz4_buf.offset > inf->lz4_buf.cap / 2) {
             if (data_len > 0)
                 memmove(inf->lz4_buf.buf, inf->lz4_buf.buf + inf->lz4_buf.offset, data_len);
@@ -2993,11 +3077,6 @@ static VALUE inflater_write(VALUE self, VALUE chunk) {
             inf->lz4_buf.len = data_len;
         }
 
-        needed = inf->lz4_buf.len + slen;
-        if (needed > inf->lz4_buf.cap) {
-            inf->lz4_buf.cap = needed * 2;
-            REALLOC_N(inf->lz4_buf.buf, char, inf->lz4_buf.cap);
-        }
         memcpy(inf->lz4_buf.buf + inf->lz4_buf.len, src, slen);
         inf->lz4_buf.len += slen;
 

data/lib/multi_compress/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MultiCompress
-  VERSION = "0.3.3"
+  VERSION = "0.3.4"
 end

data/lib/multi_compress.rb

@@ -97,17 +97,19 @@ module MultiCompress
   end
 
   def self.zstd(data, level: nil)
-    compress(data, algo: :zstd, **level_opts(level))
+    compress(data, algo: :zstd, level: level)
   end
 
   def self.lz4(data, level: nil, format: nil)
-    opts = level_opts(level)
-    opts[:format] = format if format
-    compress(data, algo: :lz4, **opts)
+    if format
+      compress(data, algo: :lz4, level: level, format: format)
+    else
+      compress(data, algo: :lz4, level: level)
+    end
   end
 
   def self.brotli(data, level: nil)
-    compress(data, algo: :brotli, **level_opts(level))
+    compress(data, algo: :brotli, level: level)
   end
 
   def self.decompress(data, **opts)
@@ -119,9 +121,11 @@ module MultiCompress
   end
 
   def self.lz4_decompress(data, format: nil)
-    opts = { algo: :lz4 }
-    opts[:format] = format if format
-    decompress(data, **opts)
+    if format
+      decompress(data, algo: :lz4, format: format)
+    else
+      decompress(data, algo: :lz4)
+    end
   end
 
   def self.brotli_decompress(data)
@@ -132,17 +136,13 @@ module MultiCompress
     EXTENSION_MAP[File.extname(path).downcase]
   end
 
-  def self.level_opts(level)
-    level ? { level: level } : {}
-  end
-
   def self.resolved_one_shot_options(opts)
-    resolved = opts.dup
-    resolved[:max_output_size] = config.max_output_size unless resolved.key?(:max_output_size)
-    resolved
+    return opts.merge(max_output_size: config.max_output_size) unless opts.key?(:max_output_size)
+
+    opts
   end
 
-  private_class_method :level_opts, :resolved_one_shot_options
+  private_class_method :resolved_one_shot_options
 
   module InflaterDefaults
     def initialize(*args, **opts)
@@ -248,6 +248,7 @@ module MultiCompress
 
   class Reader
     CHUNK_SIZE = 8192
+    BUFFER_COMPACT_THRESHOLD = 64 * 1024
 
     def self.open(path_or_io, algo: nil, dictionary: nil, **opts, &block)
       io, algo, owned = resolve_io(path_or_io, algo, mode: "rb")
@@ -264,12 +265,13 @@ module MultiCompress
     end
 
     def initialize(io, algo: nil, dictionary: nil, **opts)
-      @io       = io
-      @inflater = Inflater.new(algo: algo, dictionary: dictionary, **opts)
-      @closed   = false
-      @owned_io = false
-      @buffer   = +""
-      @eof      = false
+      @io          = io
+      @inflater    = Inflater.new(algo: algo, dictionary: dictionary, **opts)
+      @closed      = false
+      @owned_io    = false
+      @buffer      = +"".b
+      @buffer_pos  = 0
+      @eof         = false
     end
 
     def read(length = nil)
@@ -281,12 +283,12 @@ module MultiCompress
 
     def gets(separator = "\n")
       ensure_open!
-      return nil if @eof && @buffer.empty?
+      return nil if @eof && buffer_empty?
 
-      fill_buffer_until { @buffer.include?(separator) }
+      fill_buffer_until { buffer_includes?(separator) }
 
-      return extract_line(separator) if @buffer.include?(separator)
-      return consume_buffer unless @buffer.empty?
+      return extract_line(separator) if buffer_includes?(separator)
+      return consume_buffer unless buffer_empty?
 
       nil
     end
@@ -305,7 +307,7 @@ module MultiCompress
     end
 
     def eof?
-      @eof && @buffer.empty?
+      @eof && buffer_empty?
     end
 
     def each_line
@@ -334,11 +336,44 @@ module MultiCompress
       raise StreamError, "reader is closed" if @closed
     end
 
+    def buffer_size
+      @buffer.bytesize - @buffer_pos
+    end
+
+    def buffer_empty?
+      @buffer_pos >= @buffer.bytesize
+    end
+
+    def buffer_append(data)
+      compact_buffer_if_needed
+      @buffer << data
+    end
+
+    def compact_buffer_if_needed
+      return if @buffer_pos == 0
+
+      total = @buffer.bytesize
+      return unless @buffer_pos >= BUFFER_COMPACT_THRESHOLD && @buffer_pos * 2 >= total
+
+      @buffer = @buffer.byteslice(@buffer_pos, total - @buffer_pos)
+      @buffer_pos = 0
+    end
+
+    def buffer_includes?(separator)
+      idx = @buffer.index(separator, @buffer_pos)
+      !idx.nil?
+    end
+
     def read_all
-      return nil if @eof && @buffer.empty?
+      return nil if @eof && buffer_empty?
 
-      result = @buffer.dup
+      result = if buffer_empty?
+        +"".b
+      else
+        @buffer.byteslice(@buffer_pos, @buffer.bytesize - @buffer_pos) || +"".b
+      end
       @buffer.clear
+      @buffer_pos = 0
 
       until @eof
         chunk = read_compressed_chunk
@@ -355,15 +390,16 @@ module MultiCompress
     end
 
     def read_exactly(length)
-      return nil if @eof && @buffer.empty?
+      return nil if @eof && buffer_empty?
 
-      fill_buffer_until { @buffer.bytesize >= length }
+      fill_buffer_until { buffer_size >= length }
 
-      if @buffer.bytesize >= length
-        result  = @buffer[0, length]
-        @buffer = @buffer[length..]
+      if buffer_size >= length
+        result = @buffer.byteslice(@buffer_pos, length)
+        @buffer_pos += length
+        compact_buffer_if_needed
         result
-      elsif !@buffer.empty?
+      elsif !buffer_empty?
         consume_buffer
       end
     end
@@ -376,20 +412,23 @@ module MultiCompress
           break
         end
         decompressed = @inflater.write(chunk)
-        @buffer << decompressed if decompressed
+        buffer_append(decompressed) if decompressed
       end
     end
 
     def extract_line(separator)
-      idx = @buffer.index(separator)
-      result = @buffer[0, idx + separator.bytesize]
-      @buffer = @buffer[(idx + separator.bytesize)..] || +""
+      idx = @buffer.index(separator, @buffer_pos)
+      end_pos = idx + separator.bytesize
+      result = @buffer.byteslice(@buffer_pos, end_pos - @buffer_pos)
+      @buffer_pos = end_pos
+      compact_buffer_if_needed
       result
     end
 
     def consume_buffer
-      result  = @buffer
-      @buffer = +""
+      result = @buffer.byteslice(@buffer_pos, @buffer.bytesize - @buffer_pos) || +"".b
+      @buffer.clear
+      @buffer_pos = 0
       result
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multi_compress
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Roman Haydarov
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-30 00:00:00.000000000 Z
+date: 2026-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler