tinybits 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c60cd0920984c872762a71f597b65c2c9712791ea0b3e8a54a20282536f8a6e
-  data.tar.gz: 40d72eea34b0e6c07f6747955edca46dfed52a9a624f8e19d2e93ae8a7f6dfa6
+  metadata.gz: d36aff126ea60de67bc4050896a329f39e53a155cf50f4b12f35602757dc4ab1
+  data.tar.gz: 610909057ad8ae5f8fa0dab0e1dd5ee03105b7fb0036b11e3ac22eea97522cee
 SHA512:
-  metadata.gz: 1a4086aa883b8af346ff003b3b414aea443fd56caff4f1da6730cf974365886ae78eceb84c1d37d9d4aea026121cafc0eb5febb4ee3c1ff03e48dc0b646fb203
-  data.tar.gz: 063a02e5715833da64c75fd54e6a353e3d6cf89b61d704085edd3791a6b31131398174b3d27e5e1189fd1e0137644aed2358a26e0b2576b717f8775b5a951c0b
+  metadata.gz: fb4ce2802d01a5a62a119edc8f8c7ea06aff12edf0eff1d9b0d3a758d096e36c40c49c3ca52c22368da199b65b75900db1a5296580e38b9d2c00da62b80d6535
+  data.tar.gz: 16355ff8ae048bf3ae8d50eb2067cc42e01e21da1a396713d3a74e831e163658f3631dc19c09fc52c867fcf4b24bd29399029f44fabe9ac92be99eaccaa66a9b

data/ext/tinybits/tinybits.h

@@ -1,6 +1,6 @@
 /**
  * TinyBits Amalgamated Header
- * Generated on: Fri 30 Oct 2026 12:29:55 PM CET
+ * Generated on: Tue 03 Nov 2026 03:43:24 AM CET
  */
 
 #ifndef TINY_BITS_H
@@ -15,7 +15,8 @@
 #include <math.h>
 #include <stdio.h>
 
-#define TB_HASH_SIZE 128
+#define TB_HASH_BITS 7
+#define TB_HASH_SIZE (1 << TB_HASH_BITS)
 #define TB_HASH_CACHE_SIZE 256
 #define MAX_BYTES 9
 #define TB_DDP_STR_LEN_MAX 128
@@ -84,7 +85,7 @@ typedef struct HashTable {
     uint32_t next_id;
     uint32_t cache_size;
     uint32_t cache_pos;
-    uint8_t bins[TB_HASH_SIZE];
+    uint16_t bins[TB_HASH_SIZE]; // 1-based cache index, must hold TB_HASH_CACHE_SIZE
 } HashTable;
 
 static inline uint32_t fast_hash_32(const char* str, uint16_t len) {
@@ -97,6 +98,11 @@ static inline uint32_t fast_hash_32(const char* str, uint16_t len) {
     return hash;
 }
 
+// bin index from the top bits of a multiplicative mix (the low bits alone are just the last byte)
+static inline uint32_t tb_hash_bin(uint32_t hash_code) {
+    return (hash_code * 0x9E3779B1u) >> (32 - TB_HASH_BITS);
+}
+
 static inline int encode_varint(uint64_t value, uint8_t* buffer) {
     if (value <= 240) {
         buffer[0] = (uint8_t)value;  // 1 byte
@@ -413,7 +419,9 @@ typedef struct tiny_bits_packer {
     size_t capacity;         // Total allocated size of the buffer
     size_t current_pos;      // Current position in the buffer (write position)
     HashTable encode_table; // Add the hash table here
-    HashTable dictionary;
+    size_t dict_len;         // dictionary bytes stored at the start of buffer
+    uint32_t dict_count;     // dedupe cache entries taken by the dictionary
+    uint16_t dict_bins[TB_HASH_SIZE]; // encode_table.bins right after loading the dictionary
     uint8_t features;
     // Add any other encoder-specific state here if needed (e.g., string deduplication table later)
 } tiny_bits_packer;
@@ -451,6 +459,9 @@ tiny_bits_packer *tiny_bits_packer_create(size_t initial_capacity, uint8_t featu
     encoder->capacity = initial_capacity;
     encoder->current_pos = 0;
     encoder->features = features;
+    encoder->dict_len = 0;
+    encoder->dict_count = 0;
+    memset(encoder->dict_bins, 0, sizeof(encoder->dict_bins));
 
     // Only allocate hash table if deduplication is enabled
     if (features & TB_FEATURE_STRING_DEDUPE) {
@@ -464,7 +475,7 @@ tiny_bits_packer *tiny_bits_packer_create(size_t initial_capacity, uint8_t featu
         encoder->encode_table.cache_size = TB_HASH_CACHE_SIZE;
         encoder->encode_table.cache_pos = 0;
         encoder->encode_table.next_id = 0;
-        memset(encoder->encode_table.bins, 0, TB_HASH_SIZE * sizeof(uint8_t));
+        memset(encoder->encode_table.bins, 0, sizeof(encoder->encode_table.bins));
     } else {
         encoder->encode_table.cache = NULL;
         encoder->encode_table.cache_size = 0;
@@ -482,15 +493,82 @@ tiny_bits_packer *tiny_bits_packer_create(size_t initial_capacity, uint8_t featu
  *
  * @note This function allows for more efficient packing by reusing the same packer object
  */
-inline void tiny_bits_packer_reset(tiny_bits_packer *encoder) {
+static inline void tiny_bits_packer_reset(tiny_bits_packer *encoder) {
     if (!encoder) return;
-    encoder->current_pos = 0;  
+    encoder->current_pos = encoder->dict_len;
     if (encoder->features & TB_FEATURE_STRING_DEDUPE) {
         encoder->encode_table.next_id = 0;
-        encoder->encode_table.cache_pos = 0;
-        memset(encoder->encode_table.bins, 0, TB_HASH_SIZE * sizeof(uint8_t));
+        encoder->encode_table.cache_pos = encoder->dict_count;
+        memcpy(encoder->encode_table.bins, encoder->dict_bins, sizeof(encoder->dict_bins));
     }
-    
+}
+
+/**
+ * @brief Loads an external dictionary of strings used for deduplication
+ *
+ * @param encoder The packer instance (must have TB_FEATURE_STRING_DEDUPE)
+ * @param strs Array of string pointers (order is significant)
+ * @param lens Array of string lengths
+ * @param count Number of strings
+ * @return 1 on success, 0 on error
+ *
+ * @note The dictionary is not written to the output, the unpacker must be given the same dictionary.
+ * Strings are copied, and the packer is reset. Strings are assigned ids exactly as if they were packed
+ * (in order) at the start of every output, so only the first 256 dedupable strings (2-128 bytes) count.
+ */
+static inline int tiny_bits_packer_set_dict(tiny_bits_packer *encoder, const char **strs, const size_t *lens, size_t count) {
+    if (!encoder || !(encoder->features & TB_FEATURE_STRING_DEDUPE)) return 0;
+    HashTable *table = &encoder->encode_table;
+    encoder->dict_len = 0;
+    encoder->dict_count = 0;
+    memset(encoder->dict_bins, 0, sizeof(encoder->dict_bins));
+    tiny_bits_packer_reset(encoder);
+
+    for (size_t i = 0; i < count && table->cache_pos < TB_HASH_CACHE_SIZE; i++) {
+        const char *str = strs[i];
+        size_t len = lens[i];
+        if (len < 2 || len > 128) continue;
+        uint32_t hash_code = fast_hash_32(str, len);
+        uint32_t hash = tb_hash_bin(hash_code);
+        uint16_t index = table->bins[hash];
+        while (index > 0) {
+            HashEntry entry = table->cache[index - 1];
+            if (hash_code == entry.hash && len == entry.length
+                && memcmp(str, encoder->buffer + entry.offset, len) == 0) break;
+            index = entry.next_index;
+        }
+        if (index > 0) continue; // duplicate, packing it would emit a ref and take no id
+
+        uint8_t *buffer = tiny_bits_packer_ensure_capacity(encoder, len);
+        if (!buffer) return 0;
+        memcpy(buffer, str, len);
+        HashEntry *new_entry = &table->cache[table->cache_pos++];
+        new_entry->hash = hash_code;
+        new_entry->length = len;
+        new_entry->offset = encoder->current_pos;
+        new_entry->next_index = table->bins[hash];
+        table->bins[hash] = table->cache_pos;
+        encoder->current_pos += len;
+    }
+
+    encoder->dict_len = encoder->current_pos;
+    encoder->dict_count = table->cache_pos;
+    memcpy(encoder->dict_bins, table->bins, sizeof(encoder->dict_bins));
+    return 1;
+}
+
+/**
+ * @brief Returns a pointer to the packed output (excludes the dictionary)
+ */
+static inline const unsigned char *tiny_bits_packer_data(const tiny_bits_packer *encoder) {
+    return encoder->buffer + encoder->dict_len;
+}
+
+/**
+ * @brief Returns the size of the packed output (excludes the dictionary)
+ */
+static inline size_t tiny_bits_packer_size(const tiny_bits_packer *encoder) {
+    return encoder->current_pos - encoder->dict_len;
 }
 
 /**
@@ -715,8 +793,8 @@ static inline int pack_str(tiny_bits_packer *encoder, char* str, uint32_t str_le
     uint32_t hash = 0;
     if ((encoder->features & TB_FEATURE_STRING_DEDUPE) && str_len >= 2 && str_len <= 128) {
         hash_code = fast_hash_32(str, str_len);
-        hash = hash_code % TB_HASH_SIZE;
-        uint8_t index = encoder->encode_table.bins[hash];
+        hash = tb_hash_bin(hash_code);
+        uint16_t index = encoder->encode_table.bins[hash];
         while (index > 0) {
             HashEntry entry = encoder->encode_table.cache[index - 1];
             if (hash_code == entry.hash 
@@ -940,7 +1018,8 @@ typedef struct tiny_bits_unpacker {
     } *strings;           // Array of decoded strings
     size_t strings_size;  // Capacity of strings array
     size_t strings_count; // Number of strings stored
-    HashTable dictionary;
+    char *dict;           // copy of the dictionary bytes, strings[0..dict_count) point into it
+    size_t dict_count;    // number of strings entries taken by the dictionary
 } tiny_bits_unpacker;
 
 /**
@@ -962,9 +1041,63 @@ tiny_bits_unpacker *tiny_bits_unpacker_create(void) {
         return NULL;
     }
     decoder->strings_count = 0;
+    decoder->dict = NULL;
+    decoder->dict_count = 0;
     return decoder;
 }
 
+/**
+ * @brief Loads an external dictionary of strings, must match the one given to the packer
+ *
+ * @param decoder The unpacker instance
+ * @param strs Array of string pointers (order is significant)
+ * @param lens Array of string lengths
+ * @param count Number of strings
+ * @return 1 on success, 0 on error
+ *
+ * @note Strings are copied. Dictionary strings are returned as dedupe references, with ids 1..n
+ * (see unpack_value), so client code can map them to pre-built objects.
+ */
+static inline int tiny_bits_unpacker_set_dict(tiny_bits_unpacker *decoder, const char **strs, const size_t *lens, size_t count) {
+    if (!decoder) return 0;
+    size_t total = 0;
+    for (size_t i = 0; i < count; i++) total += lens[i];
+    char *dict = (char *)malloc(total ? total : 1);
+    if (!dict) return 0;
+    free(decoder->dict);
+    decoder->dict = dict;
+    decoder->dict_count = 0;
+    decoder->strings_count = 0;
+
+    size_t offset = 0;
+    for (size_t i = 0; i < count && decoder->dict_count < TB_HASH_CACHE_SIZE; i++) {
+        size_t len = lens[i];
+        if (len < 2 || len > 128) continue;
+        int duplicate = 0;
+        for (size_t j = 0; j < decoder->dict_count; j++) {
+            if (decoder->strings[j].length == len && memcmp(decoder->strings[j].str, strs[i], len) == 0) {
+                duplicate = 1;
+                break;
+            }
+        }
+        if (duplicate) continue;
+        if (decoder->dict_count >= decoder->strings_size) {
+            size_t new_size = decoder->strings_size * 2;
+            void *new_strings = realloc(decoder->strings, new_size * sizeof(*decoder->strings));
+            if (!new_strings) return 0;
+            decoder->strings = new_strings;
+            decoder->strings_size = new_size;
+        }
+        memcpy(dict + offset, strs[i], len);
+        decoder->strings[decoder->dict_count].str = dict + offset;
+        decoder->strings[decoder->dict_count].length = len;
+        decoder->dict_count++;
+        offset += len;
+    }
+    decoder->strings_count = decoder->dict_count;
+    return 1;
+}
+
 /**
  * @breif Provides a buffer to the unpacker for unpacking
  * 
@@ -982,7 +1115,7 @@ static inline void tiny_bits_unpacker_set_buffer(tiny_bits_unpacker *decoder, co
     decoder->buffer = buffer;
     decoder->size = size;
     decoder->current_pos = 0;
-    decoder->strings_count = 0;
+    decoder->strings_count = decoder->dict_count;
 }
 
 /**
@@ -995,7 +1128,7 @@ static inline void tiny_bits_unpacker_set_buffer(tiny_bits_unpacker *decoder, co
 static inline void tiny_bits_unpacker_reset(tiny_bits_unpacker *decoder) {
     if (!decoder) return;
     decoder->current_pos = 0;
-    decoder->strings_count = 0;
+    decoder->strings_count = decoder->dict_count;
 }
 
 
@@ -1010,6 +1143,7 @@ void tiny_bits_unpacker_destroy(tiny_bits_unpacker *decoder) {
     if (decoder->strings) {
         free(decoder->strings);
     }
+    free(decoder->dict);
     free(decoder);
 }
 
@@ -1215,39 +1349,31 @@ static inline enum tiny_bits_type unpack_value(tiny_bits_unpacker *decoder, tiny
     }
 
     uint8_t tag = decoder->buffer[decoder->current_pos++];
-    // Dispatch based on tag
-    if ((tag & TB_INT_TAG) == TB_INT_TAG) { // Integers
+    // Dispatch on tag ranges, each tag group starts at a power of two
+    if (tag >= TB_INT_TAG) { // Integers
         return _unpack_int(decoder, tag, value);
-    } else if ((tag & TB_STR_TAG) == TB_STR_TAG) { // Strings
+    } else if (tag >= TB_STR_TAG) { // Strings
         return _unpack_str(decoder, tag, value);
-    } else if (tag == TB_NIL_TAG) {
-        return TINY_BITS_NULL;
-    } else if (tag == TB_NAN_TAG) {
-        return TINY_BITS_NAN;
-    } else if (tag == TB_INF_TAG) {
-        return TINY_BITS_INF;
-    } else if (tag == TB_NNF_TAG) {
-        return TINY_BITS_N_INF;
-    } else if ((tag & TB_DBL_TAG) == TB_DBL_TAG) { // Doubles
+    } else if (tag >= TB_DBL_TAG) { // Doubles
+        if (tag == TB_NAN_TAG) return TINY_BITS_NAN;
+        if (tag == TB_INF_TAG) return TINY_BITS_INF;
+        if (tag == TB_NNF_TAG) return TINY_BITS_N_INF;
         return _unpack_double(decoder, tag, value);
-    } else if ((tag & TB_MAP_TAG) == TB_MAP_TAG) { // Maps
+    } else if (tag >= TB_MAP_TAG) { // Maps
         return _unpack_map(decoder, tag, value);
-    } else if ((tag & TB_ARR_TAG) == TB_ARR_TAG) { // Arrays
+    } else if (tag >= TB_ARR_TAG) { // Arrays
         return _unpack_arr(decoder, tag, value);
-    } else if (tag == TB_BLB_TAG) { // Blob
-        return _unpack_blob(decoder, tag, value);
-    } else if (tag == TB_DTM_TAG) {
-        return _unpack_datetime(decoder, tag, value);
-    } else if (tag == TB_SEP_TAG) {
-        return TINY_BITS_SEP;
-    } else if (tag == TB_EXT_TAG) {
-        return TINY_BITS_EXT;
-    } else if (tag == TB_TRU_TAG) {
-        return TINY_BITS_TRUE;
-    } else if (tag == TB_FLS_TAG) {
-        return TINY_BITS_FALSE;
     }
-    return TINY_BITS_ERROR; // Unknown tag
+    switch (tag) {
+        case TB_NIL_TAG: return TINY_BITS_NULL;
+        case TB_TRU_TAG: return TINY_BITS_TRUE;
+        case TB_FLS_TAG: return TINY_BITS_FALSE;
+        case TB_BLB_TAG: return _unpack_blob(decoder, tag, value);
+        case TB_DTM_TAG: return _unpack_datetime(decoder, tag, value);
+        case TB_SEP_TAG: return TINY_BITS_SEP;
+        case TB_EXT_TAG: return TINY_BITS_EXT;
+    }
+    return TINY_BITS_ERROR; // Unknown tag (TB_NXT_TAG)
 }
 
 

data/ext/tinybits/tinybits_ext.c

@@ -25,7 +25,7 @@ VALUE rb_cUnpacker;
 
 // Forward declarations
 static VALUE rb_packer_alloc(VALUE klass);
-static VALUE rb_packer_init(VALUE self);
+static VALUE rb_packer_init(int argc, VALUE* argv, VALUE self);
 static VALUE rb_unpacker_alloc(VALUE klass);
 static VALUE rb_unpacker_init(int argc, VALUE* argv, VALUE self);
 static VALUE rb_pack(VALUE self, VALUE obj);
@@ -37,6 +37,8 @@ typedef struct {
     int error_occurred; // 0 for no error, 1 if callback encountered an error
 } HashIterContext;
 
+static int binary_encindex; // rb_ascii8bit_encindex(), set in Init
+
 // Forward declaration for the recursive function
 static int pack_ruby_object_recursive(tiny_bits_packer* packer, VALUE obj, VALUE context);
 
@@ -71,7 +73,7 @@ static void packer_free(void* data) {
     if (packer_data->packer) {
         tiny_bits_packer_destroy(packer_data->packer);
     }
-    free(packer_data);
+    xfree(packer_data);
 }
 
 static size_t packer_memsize(const void* data) {
@@ -90,21 +92,52 @@ static VALUE rb_packer_alloc(VALUE klass) {
     return TypedData_Wrap_Struct(klass, &packer_data_type, packer_data);
 }
 
-/* 
+// Calls set_dict(target, strs, lens, count) with the #to_s of each dictionary member
+#define WITH_DICT_STRINGS(dict, set_dict, target) do {                                  \
+    Check_Type(dict, T_ARRAY);                                                         \
+    long dict_n = RARRAY_LEN(dict);                                                    \
+    VALUE dict_strs = rb_ary_new_capa(dict_n);                                         \
+    for (long i = 0; i < dict_n; i++) {                                                \
+        rb_ary_push(dict_strs, rb_obj_as_string(rb_ary_entry(dict, i)));               \
+    }                                                                                  \
+    VALUE dict_buf;                                                                    \
+    const char **dict_ptrs = ALLOCV(dict_buf, dict_n * (sizeof(char *) + sizeof(size_t)) + 1); \
+    size_t *dict_lens = (size_t *)(dict_ptrs + dict_n);                                \
+    for (long i = 0; i < dict_n; i++) {                                                \
+        VALUE s = RARRAY_AREF(dict_strs, i);                                           \
+        dict_ptrs[i] = RSTRING_PTR(s);                                                 \
+        dict_lens[i] = RSTRING_LEN(s);                                                 \
+    }                                                                                  \
+    int dict_ok = set_dict(target, dict_ptrs, dict_lens, dict_n);                      \
+    ALLOCV_END(dict_buf);                                                              \
+    RB_GC_GUARD(dict_strs);                                                            \
+    if (!dict_ok) rb_raise(rb_eRuntimeError, "Failed to set dictionary");              \
+} while (0)
+
+/*
  * Document-method: initialize
  *
  * Initializes a new Packer object
  *
+ * @param dict [Array] strings (or objects responding to #to_s) to deduplicate against without
+ *   storing them in the output (order is significant), the unpacker must use the same dict
  * @return [Packer] The initialized packer object.
  */
-static VALUE rb_packer_init(VALUE self) {
+static VALUE rb_packer_init(int argc, VALUE* argv, VALUE self) {
     PackerData* packer_data;
     TypedData_Get_Struct(self, PackerData, &packer_data_type, packer_data);
 
+    VALUE opts = Qnil;
+    rb_scan_args(argc, argv, ":", &opts);
+    VALUE dict = NIL_P(opts) ? Qnil : rb_hash_lookup(opts, ID2SYM(rb_intern("dict")));
+
     packer_data->packer = tiny_bits_packer_create(256, (TB_FEATURE_STRING_DEDUPE | TB_FEATURE_COMPRESS_FLOATS) | 0); // Initial capacity and features
     if (!packer_data->packer) {
         rb_raise(rb_eRuntimeError, "Failed to initialize packer");
     }
+    if (!NIL_P(dict)) {
+        WITH_DICT_STRINGS(dict, tiny_bits_packer_set_dict, packer_data->packer);
+    }
     return self;
 }
 
@@ -112,7 +145,7 @@ static VALUE rb_packer_init(VALUE self) {
 static inline int pack_ruby_object_recursive(tiny_bits_packer* packer, VALUE obj, VALUE context) {
     switch (TYPE(obj)) {
         case T_STRING: {
-            if (ENCODING_GET_INLINED(obj) == rb_ascii8bit_encindex())
+            if (ENCODING_GET_INLINED(obj) == binary_encindex)
                 return pack_blob(packer, RSTRING_PTR(obj), RSTRING_LEN(obj));
             return pack_str(packer, RSTRING_PTR(obj), RSTRING_LEN(obj));
         }
@@ -128,13 +161,12 @@ static inline int pack_ruby_object_recursive(tiny_bits_packer* packer, VALUE obj
             int written = pack_arr(packer, len); 
             if (written <= 0) return 0; // Error check based on tiny_bits API
             for (long i = 0; i < len; i++) {
-                if(!pack_ruby_object_recursive(packer, rb_ary_entry(obj, i), context)) return 0; // Propagate error
+                if(!pack_ruby_object_recursive(packer, RARRAY_AREF(obj, i), context)) return 0; // Propagate error
             }
             return 1; // Success
         }
         case T_FIXNUM: {
-            int64_t val = NUM2LONG(obj); // Assumes fits in int64_t
-            return pack_int(packer, val);
+            return pack_int(packer, (int64_t)FIX2LONG(obj));
         }
         case T_BIGNUM: {
             // within int64: a Bignum only past the Fixnum range (2**62)
@@ -145,8 +177,7 @@ static inline int pack_ruby_object_recursive(tiny_bits_packer* packer, VALUE obj
             return pack_int(packer, (int64_t)NUM2LL(obj));
         }
         case T_FLOAT: {
-            double val = NUM2DBL(obj);
-            return pack_double(packer, val);
+            return pack_double(packer, RFLOAT_VALUE(obj));
         }
         case T_NIL:
             return pack_null(packer);
@@ -206,7 +237,7 @@ static VALUE rb_pack(VALUE self, VALUE obj) {
         rb_raise(rb_eRuntimeError, "Failed to pack object (unsupported type or packing error)");
     }
 
-    VALUE result = rb_str_new((const char*)packer_data->packer->buffer, packer_data->packer->current_pos);
+    VALUE result = rb_str_new((const char*)tiny_bits_packer_data(packer_data->packer), tiny_bits_packer_size(packer_data->packer));
     rb_obj_freeze(result);
     return result;
 }
@@ -266,10 +297,10 @@ static VALUE rb_to_s(VALUE self){
         rb_raise(rb_eRuntimeError, "Packer not initialized");
     }
 
-    if(packer_data->packer->current_pos == 0){
+    if(tiny_bits_packer_size(packer_data->packer) == 0){
         return rb_str_new("", 0);
     }
-    VALUE result = rb_str_new((const char*)packer_data->packer->buffer, packer_data->packer->current_pos);
+    VALUE result = rb_str_new((const char*)tiny_bits_packer_data(packer_data->packer), tiny_bits_packer_size(packer_data->packer));
     rb_obj_freeze(result);
     return result;
 }
@@ -299,19 +330,27 @@ static VALUE rb_reset(VALUE self){
 typedef struct {
     tiny_bits_unpacker* unpacker;
     int symbolize_keys; // map keys that are strings come back as symbols
-    //size_t strings_index;
-    //VALUE ruby_strings[TB_HASH_CACHE_SIZE];
+    VALUE strings;      // Ruby strings by dedupe id, dictionary strings first
+    VALUE buffer;       // the buffer set by #buffer=, kept alive while popping
 } UnpackerData;
 
+static void unpacker_mark(void* data) {
+    UnpackerData* unpacker_data = (UnpackerData*)data;
+    rb_gc_mark_movable(unpacker_data->strings);
+    rb_gc_mark(unpacker_data->buffer); // pinned, the C unpacker points into it (embedded strings move)
+}
+
+static void unpacker_compact(void* data) {
+    UnpackerData* unpacker_data = (UnpackerData*)data;
+    unpacker_data->strings = rb_gc_location(unpacker_data->strings);
+}
+
 static void unpacker_free(void* data) {
     UnpackerData* unpacker_data = (UnpackerData*)data;
     if (unpacker_data->unpacker) {
         tiny_bits_unpacker_destroy(unpacker_data->unpacker);
     }
-    //for (size_t i = 0; i < TB_HASH_CACHE_SIZE; i++) {
-        //unpacker_data->ruby_strings[i] = Qnil;
-    //}
-    free(unpacker_data);
+    xfree(unpacker_data);
 }
 
 static size_t unpacker_memsize(const void* data) {
@@ -320,7 +359,7 @@ static size_t unpacker_memsize(const void* data) {
 
 static const rb_data_type_t unpacker_data_type = {
     "TinyBits::Unpacker",
-    {0, unpacker_free, unpacker_memsize,},
+    {unpacker_mark, unpacker_free, unpacker_memsize, unpacker_compact,},
     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 };
 
@@ -328,6 +367,8 @@ static VALUE rb_unpacker_alloc(VALUE klass) {
     UnpackerData* unpacker_data = ALLOC(UnpackerData);
     unpacker_data->unpacker = NULL;
     unpacker_data->symbolize_keys = 0;
+    unpacker_data->strings = Qnil;
+    unpacker_data->buffer = Qnil;
     return TypedData_Wrap_Struct(klass, &unpacker_data_type, unpacker_data);
 }
 
@@ -337,6 +378,7 @@ static VALUE rb_unpacker_alloc(VALUE klass) {
  * Initializes a new Unpacker.
  *
  * @param symbolize_keys [Boolean] unpack hash keys that are strings as symbols (default false)
+ * @param dict [Array] the dictionary the packer used (order is significant)
  * @return [Unpacker] The initialized unpacker object.
  */
 static VALUE rb_unpacker_init(int argc, VALUE* argv, VALUE self) {
@@ -344,9 +386,11 @@ static VALUE rb_unpacker_init(int argc, VALUE* argv, VALUE self) {
     TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
 
     VALUE opts = Qnil;
+    VALUE dict = Qnil;
     rb_scan_args(argc, argv, ":", &opts);
     if (!NIL_P(opts)) {
         unpacker_data->symbolize_keys = RTEST(rb_hash_lookup(opts, ID2SYM(rb_intern("symbolize_keys"))));
+        dict = rb_hash_lookup(opts, ID2SYM(rb_intern("dict")));
     }
 
     unpacker_data->unpacker = tiny_bits_unpacker_create();
@@ -354,10 +398,22 @@ static VALUE rb_unpacker_init(int argc, VALUE* argv, VALUE self) {
         rb_raise(rb_eRuntimeError, "Failed to initialize unpacker");
     }
     VALUE strings = rb_ary_new_capa(TB_HASH_CACHE_SIZE);
-    rb_iv_set(self, "@strings", strings);
+    if (!NIL_P(dict)) {
+        tiny_bits_unpacker* unpacker = unpacker_data->unpacker;
+        WITH_DICT_STRINGS(dict, tiny_bits_unpacker_set_dict, unpacker);
+        // dictionary strings stay in the first dict_count slots, only later ones are cleared
+        for (size_t i = 0; i < unpacker->dict_count; i++) {
+            rb_ary_push(strings, rb_enc_interned_str(unpacker->strings[i].str, unpacker->strings[i].length, rb_utf8_encoding()));
+        }
+    }
+    RB_OBJ_WRITE(self, &unpacker_data->strings, strings);
     return self;
 }
 
+static inline void clear_strings(VALUE strings, UnpackerData* unpacker_data) {
+    rb_ary_resize(strings, unpacker_data->unpacker->dict_count);
+}
+
 static inline VALUE rb_unpack_str(VALUE strings, UnpackerData* unpacker_data, tiny_bits_value value, size_t interned){
     int32_t id  = value.str_blob_val.id;
     if(id > 0)
@@ -418,14 +474,22 @@ static VALUE unpack_ruby_object(VALUE strings, UnpackerData* unpacker_data, size
         }
         case TINY_BITS_MAP: {
             VALUE hash = rb_hash_new_capa(value.length);
-            for (size_t i = 0; i < value.length; i++) {  
-                VALUE key = unpack_ruby_object(strings, unpacker_data, 0);
+            VALUE pairs[2 * 16]; // small maps are inserted in one bulk call
+            int bulk = value.length <= 16;
+            for (size_t i = 0; i < value.length; i++) {
+                VALUE key = unpack_ruby_object(strings, unpacker_data, 1);
                 if (key == Qundef) return Qundef; // Error
                 if (unpacker_data->symbolize_keys && RB_TYPE_P(key, T_STRING)) key = rb_str_intern(key);
                 VALUE val = unpack_ruby_object(strings, unpacker_data, 0);
                 if (val == Qundef) return Qundef; // Error
-                rb_hash_aset(hash, key, val);
+                if (bulk) {
+                    pairs[2 * i] = key;
+                    pairs[2 * i + 1] = val;
+                } else {
+                    rb_hash_aset(hash, key, val);
+                }
             }
+            if (bulk) rb_hash_bulk_insert(2 * value.length, pairs, hash);
             return hash;
         }
         case TINY_BITS_BLOB:
@@ -462,14 +526,18 @@ static VALUE rb_unpack(VALUE self, VALUE buffer) {
 
     tiny_bits_unpacker_set_buffer(unpacker_data->unpacker, (const unsigned char*)RSTRING_PTR(buffer), RSTRING_LEN(buffer));
 
-    VALUE array = rb_iv_get(self, "@strings");
+    VALUE array = unpacker_data->strings;
+    clear_strings(array, unpacker_data); // a failed unpack or a pop session may have left entries
     VALUE result = unpack_ruby_object(array, unpacker_data, 0);
+    RB_GC_GUARD(buffer);
+    // unpack is for one object only, a later #pop finds an empty buffer (and never reads this one)
+    unpacker_data->unpacker->current_pos = unpacker_data->unpacker->size;
 
     if (result == Qundef) {
         rb_raise(rb_eRuntimeError, "Failed to unpack data");
     }
 
-    rb_ary_clear(array);
+    clear_strings(array, unpacker_data);
 
     return result;
 }
@@ -492,13 +560,13 @@ static VALUE rb_set_buffer(VALUE self, VALUE buffer){
     }
 
     StringValue(buffer); // Ensure it's a string
+    buffer = rb_str_new_frozen(buffer); // the caller mutating their string must not move the bytes we point into
 
     tiny_bits_unpacker_set_buffer(unpacker_data->unpacker, (const unsigned char*)RSTRING_PTR(buffer), RSTRING_LEN(buffer));
 
-    // set the buffer as an instance variable to mainatin a reference to it
-    rb_iv_set(self, "@buffer", buffer);
-    VALUE array = rb_iv_get(self, "@strings");
-    rb_ary_clear(array);
+    // keep a reference to the buffer while popping
+    RB_OBJ_WRITE(self, &unpacker_data->buffer, buffer);
+    clear_strings(unpacker_data->strings, unpacker_data);
     return self;
 }
 
@@ -512,17 +580,14 @@ static VALUE rb_set_buffer(VALUE self, VALUE buffer){
  * @note Requires #buffer= to be called first.
  */
 static VALUE rb_pop(VALUE self) {
+    UnpackerData* unpacker_data;
+    TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
 
-    VALUE buffer = rb_iv_get(self, "@buffer");
-    VALUE array = rb_iv_get(self, "@strings");
-    
-    if(buffer == Qnil){
+    VALUE array = unpacker_data->strings;
+    if(unpacker_data->buffer == Qnil){
         rb_raise(rb_eRuntimeError, "No buffer is set");
     }
 
-    UnpackerData* unpacker_data;
-    TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
-
     if (!unpacker_data->unpacker) {
         rb_raise(rb_eRuntimeError, "Unpacker not initialized");
     }
@@ -530,7 +595,7 @@ static VALUE rb_pop(VALUE self) {
     tiny_bits_unpacker* unpacker = unpacker_data->unpacker;
 
     if(unpacker->current_pos >= unpacker->size){
-        rb_ary_clear(array);
+        clear_strings(array, unpacker_data);
         return Qnil;
     }
 
@@ -540,7 +605,7 @@ static VALUE rb_pop(VALUE self) {
     }
 
     if(unpacker->current_pos >= unpacker->size){
-        rb_ary_clear(array);
+        clear_strings(array, unpacker_data);
     }
 
     return result;
@@ -556,12 +621,11 @@ static VALUE rb_pop(VALUE self) {
  * @note Requires #buffer= to be called first.
  */
 static VALUE rb_finished(VALUE self){
-    VALUE buffer = rb_iv_get(self, "@buffer");
-    if(buffer == Qnil){
-        rb_raise(rb_eRuntimeError, "No buffer is set");
-    }
     UnpackerData* unpacker_data;
     TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
+    if(unpacker_data->buffer == Qnil){
+        rb_raise(rb_eRuntimeError, "No buffer is set");
+    }
 
     if (!unpacker_data->unpacker) {
         rb_raise(rb_eRuntimeError, "Unpacker not initialized");
@@ -577,12 +641,13 @@ static VALUE rb_finished(VALUE self){
 }
 
 void Init_tinybits_ext(void) {
+    binary_encindex = rb_ascii8bit_encindex();
     rb_mTinyBits = rb_define_module("TinyBits");
     rb_cPacker = rb_define_class_under(rb_mTinyBits, "Packer", rb_cObject);
     rb_cUnpacker = rb_define_class_under(rb_mTinyBits, "Unpacker", rb_cObject);
 
     rb_define_alloc_func(rb_cPacker, rb_packer_alloc);
-    rb_define_method(rb_cPacker, "initialize", rb_packer_init, 0);
+    rb_define_method(rb_cPacker, "initialize", rb_packer_init, -1);
     rb_define_method(rb_cPacker, "pack", rb_pack, 1);
     rb_define_alias(rb_cPacker, "encode", "pack");
     rb_define_alias(rb_cPacker, "dump", "pack");

data/lib/tinybits/dpacker.rb

@@ -1,57 +1,15 @@
 module TinyBits
- 
-  class DPacker
-    
+
+  # a Packer with an external dictionary, same as Packer.new(dict: dict)
+  class DPacker < Packer
+
     # creates a new TinyBits::DPacker object
     # @param dict [[String]] an array of strings to use as a dictiontary (order is significant).
     # @return [TinyBits::DPacker] the dpacker object.
     # @raise [RuntimeError] the dict is nil or if any member doesn't respnd to to_s.
     def initialize(dict)
-      @dict = dict.collect{|a| a.to_s }
-      @dict_set = false
-      @packer = Packer.new
-    end
-
-    def reset
-      @packer.reset
-      @dict_set = false
-      self
-    end
-
-    # packs an object to a binary string using the stored dictionary
-    # @param obj [Object] The Ruby object to pack.
-    # @return [String] The packed binary string (frozen).
-    # @raise [RuntimeError] If packing fails due to unsupported types or other errors.
-    def pack(object) 
-      reset
-      self << object
-      to_s
-    end
-
-    # push an object to an inceremental buffer
-    # @param obj [Object] The Ruby object to pack.
-    # @return [Integer] The number of bytes written to the buffer.
-    # @raise [RuntimeError] If packing fails due to unsupported types or other errors.
-    def <<(object)
-      if !@dict_set
-        #@packer.reset
-        @dict_size = @packer << @dict
-        @dict_set = true                
-      end
-      res = @packer << object
-      res
-    end
-
-    # return the packed buffer
-    # @return [String] The packed buffer (minus the dictionary).
-    def to_s
-      res = @packer.to_s
-      if @dict_set
-        res[@dict_size, res.bytesize - @dict_size]
-      else
-        res
-      end
+      super(dict: dict)
     end
 
   end
-end
+end

data/lib/tinybits/dunpacker.rb

@@ -1,37 +1,14 @@
 module TinyBits
 
-  class DUnpacker
-    
+  # an Unpacker with an external dictionary, same as Unpacker.new(dict: dict)
+  class DUnpacker < Unpacker
+
     # @param dict [[String]] the dictionary the packer used (order is significant).
     # @param symbolize_keys [Boolean] unpack hash keys that are strings as symbols.
     def initialize(dict, symbolize_keys: false)
-      @dict = Packer.new.pack(dict.collect{|a| a.to_s })
-      @unpacker = Unpacker.new(symbolize_keys: symbolize_keys)
-      @dict_popped = false
-    end
-
-    def unpack(buffer)
-      self.buffer = buffer
-      pop
-    end
-
-    def buffer=(buffer)
-      @unpacker.buffer = "#{@dict}#{buffer}" 
-      @dict_popped = false         
-    end
-
-    def pop
-      if !@dict_popped
-        @unpacker.pop
-        @dict_popped = true              
-      end
-      @unpacker.pop
-    end
-
-    def finished?
-      @unpacker.finished?
+      super(dict: dict, symbolize_keys: symbolize_keys)
     end
 
   end
 
-end
+end

data/lib/tinybits/version.rb

@@ -1,3 +1,3 @@
 module TinyBits
-  VERSION = '0.8.0'.freeze
+  VERSION = '0.9.0'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tinybits
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Mohamed Hassan