multi_compress 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f801c58708b34a23e3348b12d482ce3afbbf12208bdfa7d8454b20d4afbf55f4
-  data.tar.gz: 666eee26931519bf0069d451767053bc87cf91cede44cfe69a87408dc4b08736
+  metadata.gz: 404264c1ba462057374ad7a6a2a72565a16e1da607e47b1d3d91b727b4e74351
+  data.tar.gz: d23f9c3af016742a7aef3880209a2e68ba5ed2107c6c3eb3943a30802606c6ac
 SHA512:
-  metadata.gz: 39c03d5ce59b250947c9c91a61926d5ac2bdad8193cdd0b5d38bdb639e7ad38921a26e439729c0ea55cebe133a3724bd3010e3d83147aa4ce00a66909f34d7e6
-  data.tar.gz: d633ab7a2257c71d741b66c598aef4924ca99ef62dbaa01185aa1efdaaae93e7ffcaaec0205a300bc6291e279d8fafc34560a203e71da3c9ae6825c69819faa6
+  metadata.gz: 91091ad2e184758916b7c79aa0ec82a3216f5d87798a7c597b43afe25b8eaa091fd7094c5de71e4eac5e01804f7b9928734514224cd0312edd1f60187d04c91b
+  data.tar.gz: 275044013ecacca6ee300ce8b980c252048f021acc2cf51e169722830add7301dbc4e9f6942c07845ea672c5e1dd9fe8074c7840b155096c3b40b2b1e3d5ebdc

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [0.3.1]
+
+### Fixed
+- Data race fix
+
 ## [0.3.0]
 
 ### Added

data/ext/multi_compress/multi_compress.c

@@ -611,7 +611,7 @@ static inline size_t fiber_maybe_yield(size_t bytes_since_yield, size_t just_pro
     return bytes_since_yield;
 }
 
-#define DICT_CDICT_CACHE_SIZE 4
+#define DICT_CDICT_CACHE_SIZE 22
 _Static_assert(DICT_CDICT_CACHE_SIZE > 0, "CDict cache needs at least one slot");
 
 typedef struct {
@@ -623,6 +623,7 @@ struct dictionary_s {
     compress_algo_t algo;
     uint8_t *data;
     size_t size;
+    pthread_mutex_t cache_mutex;
 
     cdict_cache_entry_t cdict_cache[DICT_CDICT_CACHE_SIZE];
     int cdict_cache_count;
@@ -632,12 +633,15 @@ struct dictionary_s {
 
 static void dict_free(void *ptr) {
     dictionary_t *dict = (dictionary_t *)ptr;
+    if (!dict)
+        return;
     for (int i = 0; i < dict->cdict_cache_count; i++) {
         if (dict->cdict_cache[i].cdict)
             ZSTD_freeCDict(dict->cdict_cache[i].cdict);
     }
     if (dict->ddict)
         ZSTD_freeDDict(dict->ddict);
+    pthread_mutex_destroy(&dict->cache_mutex);
     if (dict->data)
         xfree(dict->data);
     xfree(dict);
@@ -666,47 +670,78 @@ static const rb_data_type_t dictionary_type = {
 static VALUE dict_alloc(VALUE klass) {
     dictionary_t *d = ALLOC(dictionary_t);
     memset(d, 0, sizeof(dictionary_t));
+    if (pthread_mutex_init(&d->cache_mutex, NULL) != 0) {
+        xfree(d);
+        rb_raise(eMemError, "failed to initialize dictionary cache mutex");
+    }
     return TypedData_Wrap_Struct(klass, &dictionary_type, d);
 }
 
 static ZSTD_CDict *dict_get_cdict(dictionary_t *dict, int level) {
+    ZSTD_CDict *existing = NULL;
+
+    pthread_mutex_lock(&dict->cache_mutex);
     for (int i = 0; i < dict->cdict_cache_count; i++) {
-        if (dict->cdict_cache[i].level == level)
-            return dict->cdict_cache[i].cdict;
+        if (dict->cdict_cache[i].level == level) {
+            existing = dict->cdict_cache[i].cdict;
+            break;
+        }
     }
+    pthread_mutex_unlock(&dict->cache_mutex);
+
+    if (existing)
+        return existing;
 
     ZSTD_CDict *cdict = ZSTD_createCDict(dict->data, dict->size, level);
     if (!cdict)
         return NULL;
 
+    pthread_mutex_lock(&dict->cache_mutex);
     for (int i = 0; i < dict->cdict_cache_count; i++) {
         if (dict->cdict_cache[i].level == level) {
+            existing = dict->cdict_cache[i].cdict;
+            pthread_mutex_unlock(&dict->cache_mutex);
             ZSTD_freeCDict(cdict);
-            return dict->cdict_cache[i].cdict;
+            return existing;
         }
     }
 
-    if (dict->cdict_cache_count < DICT_CDICT_CACHE_SIZE) {
-        dict->cdict_cache[dict->cdict_cache_count].level = level;
-        dict->cdict_cache[dict->cdict_cache_count].cdict = cdict;
-        dict->cdict_cache_count++;
-    } else {
-        ZSTD_CDict *old_cdict = dict->cdict_cache[0].cdict;
-        memmove(&dict->cdict_cache[0], &dict->cdict_cache[1],
-                sizeof(cdict_cache_entry_t) * (DICT_CDICT_CACHE_SIZE - 1));
-        dict->cdict_cache[DICT_CDICT_CACHE_SIZE - 1].level = level;
-        dict->cdict_cache[DICT_CDICT_CACHE_SIZE - 1].cdict = cdict;
-        if (old_cdict)
-            ZSTD_freeCDict(old_cdict);
+    if (dict->cdict_cache_count >= DICT_CDICT_CACHE_SIZE) {
+        pthread_mutex_unlock(&dict->cache_mutex);
+        ZSTD_freeCDict(cdict);
+        rb_raise(eError, "zstd dictionary cdict cache exhausted");
     }
+
+    dict->cdict_cache[dict->cdict_cache_count].level = level;
+    dict->cdict_cache[dict->cdict_cache_count].cdict = cdict;
+    dict->cdict_cache_count++;
+    pthread_mutex_unlock(&dict->cache_mutex);
     return cdict;
 }
 
 static ZSTD_DDict *dict_get_ddict(dictionary_t *dict) {
+    ZSTD_DDict *existing;
+
+    pthread_mutex_lock(&dict->cache_mutex);
+    existing = dict->ddict;
+    pthread_mutex_unlock(&dict->cache_mutex);
+    if (existing)
+        return existing;
+
+    ZSTD_DDict *created = ZSTD_createDDict(dict->data, dict->size);
+    if (!created)
+        return NULL;
+
+    pthread_mutex_lock(&dict->cache_mutex);
     if (!dict->ddict) {
-        dict->ddict = ZSTD_createDDict(dict->data, dict->size);
+        dict->ddict = created;
+        pthread_mutex_unlock(&dict->cache_mutex);
+        return created;
     }
-    return dict->ddict;
+    existing = dict->ddict;
+    pthread_mutex_unlock(&dict->cache_mutex);
+    ZSTD_freeDDict(created);
+    return existing;
 }
 
 typedef struct {
@@ -2867,7 +2902,6 @@ static VALUE inflater_write(VALUE self, VALUE chunk) {
     case ALGO_LZ4: {
         size_t data_len = inf->lz4_buf.len - inf->lz4_buf.offset;
         size_t needed = data_len + slen;
-        // TODO(v0.4): optional standard LZ4 frame format support via lz4frame.h
 
         if (inf->lz4_buf.offset > 0 && needed > inf->lz4_buf.cap) {
             if (data_len > 0)
@@ -3148,12 +3182,6 @@ static VALUE train_dictionary_internal(VALUE samples, VALUE size_val, compress_a
 }
 
 static VALUE zstd_train_dictionary(int argc, VALUE *argv, VALUE self) {
-    // #if defined(__APPLE__) && (defined(__arm64__) || defined(__aarch64__))
-    //     rb_raise(eUnsupportedError,
-    //              "Zstd dictionary training is temporarily disabled on arm64-darwin "
-    //              "because the current vendored trainer path crashes on this platform");
-    // #endif
-
     VALUE samples, opts;
     rb_scan_args(argc, argv, "1:", &samples, &opts);
     reject_algorithm_keyword(opts);

data/lib/multi_compress/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MultiCompress
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multi_compress
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Roman Haydarov