tinybits 0.3.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 340b92ea6c9e9782aa1b63c08c226d23bae56d1d5780dc46fb79f9c0a508b71b
-  data.tar.gz: 7fdeb074fe4a24e061834e1084702ee97827698eaa46f907b7b0a756b7ea05be
+  metadata.gz: 9ce29d814129fc925832887a9a4d83efd6dff20a614aab92e2dc80d10197d5aa
+  data.tar.gz: 137f8181f9d0f4c4e3d6d5abe3cb3c0014de07149e08a6e1f4d6c134cbc5c003
 SHA512:
-  metadata.gz: 0c286652db3de7817fbea5241f10221f666cc298121348dbe5718f9221c6d9ef4beed593d478851367b877bc3c66bbf1dcc359b4fb5997221790353bf1bbbcb9
-  data.tar.gz: 9b67f07cb2c1b656aa7dc843fd8441760473ba8047af09cc220508ed9e604889ab213b2f4c69855c4dbe9114190836b5603c898d0c3c4f8ce9b68b2dd77539c4
+  metadata.gz: ea777d8a1c1230512e9865e6cababb992f45d70b6c1c8d03d65567c82bda4a514ea1759341decf4c2b5da6934c23d75bc4f01ca7682a32136557382ba368d256
+  data.tar.gz: f16d288249b7cdb434e23d80823a5c5816a7b3be3989f0d0c0ce2513c7255c7f3e1df06b48f1a57681d4ccb3678328f8aa207fcdd5cbe2289cf7372f647aa52a

data/ext/tinybits/extconf.rb

@@ -1,5 +1,6 @@
 require 'mkmf'
 
+$CFLAGS << " -O3 -march=native"
 
 dir_config('tinybits_ext')
 create_makefile('tinybits_ext')

data/ext/tinybits/tinybits.h

@@ -1,6 +1,6 @@
 /**
  * TinyBits Amalgamated Header
- * Generated on: Sun May  4 03:12:10 AM CEST 2025
+ * Generated on: Tue May 13 11:39:13 PM CEST 2025
  */
 
 #ifndef TINY_BITS_H
@@ -89,8 +89,11 @@ typedef struct HashTable {
 
 static inline uint32_t fast_hash_32(const char* str, uint16_t len) {
     uint32_t hash = len;
-    hash = (hash << 16) | (((unsigned char)str[0] << 8) | (unsigned char)str[1]);
-    hash ^= (((unsigned char)str[len-2] << 24) | ((unsigned char)str[len-1] << 16));
+    hash = (hash << 24) | 
+           ((unsigned char)str[0] << 16) | 
+           ((unsigned char)str[1] << 8 ) | 
+           ((unsigned char)str[len-1]);
+    //hash ^= (((unsigned char)str[len-2] << 24) | ((unsigned char)str[len-1] << 16));
     return hash;
 }
 
@@ -173,46 +176,111 @@ static inline int varint_size(uint64_t value){
     return 9;
 }
 
-static inline uint64_t decode_varint(const uint8_t* buffer, size_t size, size_t *pos) {
+static inline int8_t decode_varint(const uint8_t* buffer, size_t size, size_t pos, uint64_t *value) {
+    if(pos >= size) return 0;
+    uint8_t prefix = buffer[pos];
+    if (prefix <= 240) {
+        *value = prefix;
+        return 1;
+    } else if (prefix >= 241 && prefix <= 248) {
+        if (pos + 1 >= size) return 0; // Not enough bytes
+        *value = 240 + 256 * (prefix - 241) + buffer[pos+1];
+        return 2;
+    } else if (prefix == 249){
+        if (pos + 2 >= size) return 0; // Not enough bytes
+        *value = 2288 + 256 * buffer[pos+1] + buffer[pos+2];
+        return 3;
+    } else if (prefix == 250){
+        if (pos + 3 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 16) | ((uint64_t)buffer[pos+2] << 8) | buffer[pos+3];
+        return 4;
+    } else if (prefix == 251){
+        if (pos + 4 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 24) | ((uint64_t)buffer[pos+2] << 16) |
+               ((uint64_t)buffer[pos+3] << 8) | buffer[pos+4];
+        return 5;    
+    } else if (prefix == 252){
+        if (pos + 5 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 32) | ((uint64_t)buffer[pos+2] << 24) |
+               ((uint64_t)buffer[pos+3] << 16) | ((uint64_t)buffer[pos+4] << 8) | buffer[pos+5];
+        return 6;        
+    } else if (prefix == 253){
+        if (pos + 6 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 40) | ((uint64_t)buffer[pos+2] << 32) |
+               ((uint64_t)buffer[pos+3] << 24) | ((uint64_t)buffer[pos+4] << 16) |
+               ((uint64_t)buffer[pos+5] << 8) | buffer[pos+6];
+        return 7;            
+    } else if (prefix == 254){
+        if (pos + 7 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 48) | ((uint64_t)buffer[pos+2] << 40) |
+               ((uint64_t)buffer[pos+3] << 32) | ((uint64_t)buffer[pos+4] << 24) |
+               ((uint64_t)buffer[pos+5] << 16) | ((uint64_t)buffer[pos+6] << 8) | buffer[pos+7];
+        return 8;            
+    } else if (prefix == 255){
+        if (pos + 8 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 56) | ((uint64_t)buffer[pos+2] << 48) |
+               ((uint64_t)buffer[pos+3] << 40) | ((uint64_t)buffer[pos+4] << 32) |
+               ((uint64_t)buffer[pos+5] << 24) | ((uint64_t)buffer[pos+6] << 16) |
+               ((uint64_t)buffer[pos+7] << 8) | buffer[pos+8];
+        return 9;            
+    } else {
+        return 0;
+    }
+
+}
+
+static inline uint64_t decode_varint_old(const uint8_t* buffer, size_t size, size_t *pos) {
+    if (*pos >= size) {
+        return 0; // not enough buffer
+    }
+
     uint8_t prefix = buffer[*pos];
     if (prefix <= 240) {
         *pos += 1;
         return prefix;
     } else if (prefix >= 241 && prefix <= 248) {
+        if (*pos + 1 >= size) return 0; // Not enough bytes
         uint64_t value = 240 + 256 * (prefix - 241) + buffer[*pos+1];
         *pos += 2;
         return value;
     } else if (prefix == 249) {
+        if (*pos + 2 >= size) return 0; // Not enough bytes
         uint64_t value = 2288 + 256 * buffer[*pos+1] + buffer[*pos+2];
         *pos += 3;
         return value;
     } else if (prefix == 250) {
+        if (*pos + 3 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 16) | ((uint64_t)buffer[*pos+2] << 8) | buffer[*pos+3];
         *pos += 4;
         return value;
     } else if (prefix == 251) {
+        if (*pos + 4 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 24) | ((uint64_t)buffer[*pos+2] << 16) |
                ((uint64_t)buffer[*pos+3] << 8) | buffer[*pos+4];
         *pos += 5;
         return value;
     } else if (prefix == 252) {
+        if (*pos + 5 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 32) | ((uint64_t)buffer[*pos+2] << 24) |
                ((uint64_t)buffer[*pos+3] << 16) | ((uint64_t)buffer[*pos+4] << 8) | buffer[*pos+5];
         *pos += 6;
         return value;
     } else if (prefix == 253) {
+        if (*pos + 6 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 40) | ((uint64_t)buffer[*pos+2] << 32) |
                ((uint64_t)buffer[*pos+3] << 24) | ((uint64_t)buffer[*pos+4] << 16) |
                ((uint64_t)buffer[*pos+5] << 8) | buffer[*pos+6];
         *pos += 7;
         return value;       
     } else if (prefix == 254) {
+        if (*pos + 7 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 48) | ((uint64_t)buffer[*pos+2] << 40) |
                ((uint64_t)buffer[*pos+3] << 32) | ((uint64_t)buffer[*pos+4] << 24) |
                ((uint64_t)buffer[*pos+5] << 16) | ((uint64_t)buffer[*pos+6] << 8) | buffer[*pos+7];
         *pos += 8;
         return value;       
     } else if (prefix == 255) {
+        if (*pos + 8 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 56) | ((uint64_t)buffer[*pos+2] << 48) |
                ((uint64_t)buffer[*pos+3] << 40) | ((uint64_t)buffer[*pos+4] << 32) |
                ((uint64_t)buffer[*pos+5] << 24) | ((uint64_t)buffer[*pos+6] << 16) |
@@ -396,6 +464,7 @@ tiny_bits_packer *tiny_bits_packer_create(size_t initial_capacity, uint8_t featu
         encoder->encode_table.cache_size = TB_HASH_CACHE_SIZE;
         encoder->encode_table.cache_pos = 0;
         encoder->encode_table.next_id = 0;
+        memset(encoder->encode_table.bins, 0, TB_HASH_SIZE * sizeof(uint8_t));
     } else {
         encoder->encode_table.cache = NULL;
         encoder->encode_table.cache_size = 0;
@@ -652,7 +721,7 @@ static inline int pack_str(tiny_bits_packer *encoder, char* str, uint32_t str_le
             HashEntry entry = encoder->encode_table.cache[index - 1];
             if (hash_code == entry.hash 
                 && str_len == entry.length
-                && (str_len <= 4 || (fast_memcmp(str, encoder->buffer + entry.offset, str_len) == 0) )) {
+                && fast_memcmp(str, encoder->buffer + entry.offset, str_len) == 0 ) {
                 id = index - 1;
                 found = 1;
                 break;
@@ -950,17 +1019,23 @@ static inline enum tiny_bits_type _unpack_int(tiny_bits_unpacker *decoder, uint8
             value->int_val = tag - 128;
             return TINY_BITS_INT;
         } else if (tag == 248) { // Positive with continuation
-            uint64_t val = decode_varint(decoder->buffer, decoder->size, &pos);
+            uint8_t read;
+            uint64_t val;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &val);
+            if(read == 0) return TINY_BITS_ERROR;
             value->int_val = val + 120;
-            decoder->current_pos = pos;
+            decoder->current_pos += read;
             return TINY_BITS_INT;
         } else if (tag > 248 && tag < 255) { // Small negative (248-254)
             value->int_val = -(tag - 248);
             return TINY_BITS_INT;
         } else { // 255: Negative with continuation
-            uint64_t val = decode_varint(decoder->buffer, decoder->size, &pos);
+            uint8_t read;
+            uint64_t val;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &val);
+            if(read == 0) return TINY_BITS_ERROR;
             value->int_val = -(val + 7);
-            decoder->current_pos = pos;
+            decoder->current_pos += read;
             return TINY_BITS_INT;
         }
 }
@@ -970,8 +1045,12 @@ static inline enum tiny_bits_type _unpack_arr(tiny_bits_unpacker *decoder, uint8
         if (tag < 0b00001111) { // Small array (0-30)
             value->length = tag & 0b00000111;
         } else { // Large array
-            value->length = decode_varint(decoder->buffer, decoder->size, &pos) + 7;
-            decoder->current_pos = pos;
+            uint8_t read;
+            uint64_t val;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &val);
+            if(read == 0) return TINY_BITS_ERROR;
+            value->length = val + 7;
+            decoder->current_pos += read;
         }
         return TINY_BITS_ARRAY;
 }
@@ -981,8 +1060,12 @@ static inline enum tiny_bits_type _unpack_map(tiny_bits_unpacker *decoder, uint8
         if (tag < 0x1F) { // Small map (0-14)
             value->length = tag & 0x0F;
         } else { // Large map
-            value->length = decode_varint(decoder->buffer, decoder->size, &pos) + 15;
-            decoder->current_pos = pos;
+            uint8_t read;
+            uint64_t val;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &val);
+            if(read == 0) return TINY_BITS_ERROR;
+            value->length = val + 15;
+            decoder->current_pos += read;
         }
         return TINY_BITS_MAP;
 }
@@ -990,28 +1073,28 @@ static inline enum tiny_bits_type _unpack_map(tiny_bits_unpacker *decoder, uint8
 static inline enum tiny_bits_type _unpack_double(tiny_bits_unpacker *decoder, uint8_t tag, tiny_bits_value *value){
         size_t pos = decoder->current_pos;
         if (tag == TB_F64_TAG) { // Raw double
+            if(pos + 8 > decoder->size) return TINY_BITS_ERROR;
             uint64_t number = decode_uint64(decoder->buffer + pos);
             value->double_val = itod_bits(number);
             decoder->current_pos += 8;
         } else { // Compressed double
-            uint64_t number = decode_varint(decoder->buffer, decoder->size, &pos);
+            uint8_t read;
+            uint64_t number;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &number);
+            if(read == 0) return TINY_BITS_ERROR;
             int order = (tag & 0x0F); 
             double fractional = (double)number / powers[order];
-            //fractional /= powers[order];
             if(tag & 0x10) fractional = -fractional;        
             value->double_val = fractional;
-            decoder->current_pos = pos;
+            decoder->current_pos += read;
         }
         return TINY_BITS_DOUBLE;
 }
 
 static inline enum tiny_bits_type _unpack_datetime(tiny_bits_unpacker *decoder, uint8_t tag, tiny_bits_value *value){
     size_t pos = decoder->current_pos;
+    if(pos + 8 > decoder->size) return TINY_BITS_ERROR;
     value->datetime_val.offset = decoder->buffer[pos] * (60*15); // convert offset back to seconds (from multiples of 15 minutes)
-    //uint8_t dbl_tag = decoder->buffer[decoder->current_pos++];
-    //tiny_bits_value dbl_val;
-    //_unpack_double(decoder, dbl_tag, &dbl_val);
-    //value->datetime_val.unixtime = dbl_val.double_val;
     uint64_t unixtime = decode_uint64(decoder->buffer + pos + 1);
     value->datetime_val.unixtime = itod_bits(unixtime);
     decoder->current_pos += 9;
@@ -1020,10 +1103,14 @@ static inline enum tiny_bits_type _unpack_datetime(tiny_bits_unpacker *decoder,
 
 static inline enum tiny_bits_type _unpack_blob(tiny_bits_unpacker *decoder, uint8_t tag, tiny_bits_value *value){
         size_t pos = decoder->current_pos;
-        size_t len = decode_varint(decoder->buffer, decoder->size, &pos);
+        size_t len;
+        size_t read; 
+        read = decode_varint(decoder->buffer, decoder->size, pos, &len);
+        if(read == 0) return TINY_BITS_ERROR;
+        if((pos + read + len) > decoder->size) return TINY_BITS_ERROR; 
         value->str_blob_val.data =  (const char *)decoder->buffer + pos;
         value->str_blob_val.length = len; 
-        decoder->current_pos = pos + len;
+        decoder->current_pos = pos + read + len;
         return TINY_BITS_BLOB;
 }
 
@@ -1032,22 +1119,35 @@ static inline enum tiny_bits_type _unpack_str(tiny_bits_unpacker *decoder, uint8
         size_t len;
         if (tag < 0x5F) { // Small string (0-30)
             len = tag & 0x1F;
+            if(pos + len > decoder->size) return TINY_BITS_ERROR;
             value->str_blob_val.data =  (const char *)decoder->buffer + pos;
             value->str_blob_val.length = len; 
-            decoder->current_pos = pos + len;
+            decoder->current_pos += len;
         } else if (tag == 0x5F) { // Large string
-            len = decode_varint(decoder->buffer, decoder->size, &pos) + 31;
+            size_t read;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &len);
+            if(read == 0) return TINY_BITS_ERROR;
+            len += 31;
+            if(pos + read + len > decoder->size) return TINY_BITS_ERROR;
             value->str_blob_val.data =  (const char *)decoder->buffer + pos;
             value->str_blob_val.length = len; 
-            decoder->current_pos = pos + len;
+            decoder->current_pos += (read + len);
         } else { // Deduplicated (small: < 0x7F, large: 0x7F)
-            uint32_t id = (tag < 0x7F) ? (tag & 0x1F) : decode_varint(decoder->buffer, decoder->size, &pos) + 31;
+            size_t id;
+            size_t read;
+            if(tag < 0x7F){
+                id = tag & 0x1F;
+            }else {
+                read = decode_varint(decoder->buffer, decoder->size, pos, &id);
+                if(read == 0) return TINY_BITS_ERROR;
+                id += 31; 
+                decoder->current_pos += read; // Update pos after varint
+            } 
             if (id >= decoder->strings_count) return TINY_BITS_ERROR;
             len = decoder->strings[id].length;
             value->str_blob_val.data = decoder->strings[id].str;
             value->str_blob_val.length = len;
-            value->str_blob_val.id = id+1;
-            decoder->current_pos = pos; // Update pos after varint
+            value->str_blob_val.id = id + 1;
             return TINY_BITS_STR;
         }
         value->str_blob_val.id = 0;

data/ext/tinybits/tinybits_ext.c

@@ -4,8 +4,23 @@
 #include "tinybits.h"
 
 // Ruby module and classes
+/*
+ * Document-module: TinyBits
+ *
+ * A Ruby extension for fast binary serialization and deserialization of Ruby objects.
+ */
 VALUE rb_mTinyBits;
+/*
+ * Document-class: TinyBits::Packer
+ *
+ * The Packer class handles serialization of Ruby objects to the TinyBits binary format.
+ */
 VALUE rb_cPacker;
+/*
+ * Document-class: TinyBits::Unpacker
+ *
+ * The Unpacker class handles deserialization of TinyBits binary format to Ruby objects.
+ */
 VALUE rb_cUnpacker;
 
 // Forward declarations
@@ -75,6 +90,13 @@ static VALUE rb_packer_alloc(VALUE klass) {
     return TypedData_Wrap_Struct(klass, &packer_data_type, packer_data);
 }
 
+/* 
+ * Document-method: initialize
+ *
+ * Initializes a new Packer object
+ *
+ * @return [Packer] The initialized packer object.
+ */
 static VALUE rb_packer_init(VALUE self) {
     PackerData* packer_data;
     TypedData_Get_Struct(self, PackerData, &packer_data_type, packer_data);
@@ -141,7 +163,17 @@ static inline int pack_ruby_object_recursive(tiny_bits_packer* packer, VALUE obj
     }
 }
 
-// keeps the public API the same.
+/*
+ * Document-method: pack
+ * 
+ * Packs a Ruby object into a binary string.
+ * Supports Ruby types: String, Array, Hash, Integer, Float, nil, true, false, Symbol, and Time.
+ * Objects can implement a `to_tinybits` method to provide custom serialization.
+ *
+ * @param obj [Object] The Ruby object to pack.
+ * @return [String] The packed binary string (frozen).
+ * @raise [RuntimeError] If packing fails due to unsupported types or other errors.
+ */
 static VALUE rb_pack(VALUE self, VALUE obj) {
     PackerData* packer_data;
     TypedData_Get_Struct(self, PackerData, &packer_data_type, packer_data);
@@ -169,6 +201,16 @@ static VALUE rb_pack(VALUE self, VALUE obj) {
     return result;
 }
 
+/*
+ * Document-method: push
+ *
+ * Appends a packed object to the current buffer.
+ * Inserts a separator when appending to non-empty buffer.
+ *
+ * @param obj [Object] The Ruby object to append.
+ * @return [Integer] The number of bytes added to the buffer.
+ * @raise [RuntimeError] If packing fails.
+ */
 static VALUE rb_push(VALUE self, VALUE obj) {
     PackerData* packer_data;
     TypedData_Get_Struct(self, PackerData, &packer_data_type, packer_data);
@@ -183,13 +225,13 @@ static VALUE rb_push(VALUE self, VALUE obj) {
     context.packer = packer_data->packer;        // Pass the current packer
     context.error_occurred = 0;   // Initialize error flag
 
-    
+    /*
     if(initial_pos > 0){
        if(!pack_separator(packer_data->packer)){
             rb_raise(rb_eRuntimeError, "Failed to pack object (multi-object packing error)");
        } 
     }
-    
+    */
     // Call the optimized recursive function
     if (!pack_ruby_object_recursive(packer_data->packer, obj, (VALUE)&context)) {
         // Error occurred during packing (might be unsupported type or tiny_bits error)
@@ -199,6 +241,13 @@ static VALUE rb_push(VALUE self, VALUE obj) {
     return INT2FIX(packer_data->packer->current_pos - initial_pos);
 }
 
+/*
+ * Document-method: to_s
+ *
+ * Returns the current packed buffer as a string.
+ *
+ * @return [String] The current packed buffer contents (frozen).
+ */
 static VALUE rb_to_s(VALUE self){
     PackerData* packer_data;
     TypedData_Get_Struct(self, PackerData, &packer_data_type, packer_data);
@@ -215,6 +264,13 @@ static VALUE rb_to_s(VALUE self){
     return result;
 }
 
+/*
+ * Document-method: reset
+ *
+ * Resets the packer's buffer to empty.
+ *
+ * @return [Packer] self
+ */
 static VALUE rb_reset(VALUE self){
     PackerData* packer_data;
     TypedData_Get_Struct(self, PackerData, &packer_data_type, packer_data);
@@ -232,8 +288,8 @@ static VALUE rb_reset(VALUE self){
 // Unpacker structure
 typedef struct {
     tiny_bits_unpacker* unpacker;
-    size_t strings_index;
-    VALUE ruby_strings[TB_HASH_CACHE_SIZE];
+    //size_t strings_index;
+    //VALUE ruby_strings[TB_HASH_CACHE_SIZE];
 } UnpackerData;
 
 static void unpacker_free(void* data) {
@@ -241,9 +297,9 @@ static void unpacker_free(void* data) {
     if (unpacker_data->unpacker) {
         tiny_bits_unpacker_destroy(unpacker_data->unpacker);
     }
-    for (size_t i = 0; i < TB_HASH_CACHE_SIZE; i++) {
-        unpacker_data->ruby_strings[i] = Qnil;
-    }
+    //for (size_t i = 0; i < TB_HASH_CACHE_SIZE; i++) {
+        //unpacker_data->ruby_strings[i] = Qnil;
+    //}
     free(unpacker_data);
 }
 
@@ -263,6 +319,13 @@ static VALUE rb_unpacker_alloc(VALUE klass) {
     return TypedData_Wrap_Struct(klass, &unpacker_data_type, unpacker_data);
 }
 
+/*
+ * Document-method: initialize
+ *
+ * Initializes a new Unpacker.
+ *
+ * @return [Unpacker] The initialized unpacker object.
+ */
 static VALUE rb_unpacker_init(VALUE self) {
     UnpackerData* unpacker_data;
     TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
@@ -271,14 +334,15 @@ static VALUE rb_unpacker_init(VALUE self) {
     if (!unpacker_data->unpacker) {
         rb_raise(rb_eRuntimeError, "Failed to initialize unpacker");
     }
-    unpacker_data->strings_index = 0;
+    VALUE strings = rb_ary_new_capa(TB_HASH_CACHE_SIZE);
+    rb_iv_set(self, "@strings", strings);
     return self;
 }
 
-static inline VALUE rb_unpack_str(UnpackerData* unpacker_data, tiny_bits_value value, size_t interned){
+static inline VALUE rb_unpack_str(VALUE strings, UnpackerData* unpacker_data, tiny_bits_value value, size_t interned){
     int32_t id  = value.str_blob_val.id;
     if(id > 0)
-        return unpacker_data->ruby_strings[id-1];
+        return rb_ary_entry(strings, id-1);
     else if(id <= 0){
         VALUE str;
         if(interned > 0){
@@ -288,14 +352,14 @@ static inline VALUE rb_unpack_str(UnpackerData* unpacker_data, tiny_bits_value v
             rb_obj_freeze(str);
         }
         if(id < 0){
-            unpacker_data->ruby_strings[abs(id)-1] = str;
+            rb_ary_push(strings, str);
         }
         return str;
     }            
     return Qundef;
 }
 
-static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
+static VALUE unpack_ruby_object(VALUE strings, UnpackerData* unpacker_data, size_t interned) {
     tiny_bits_unpacker* unpacker = unpacker_data->unpacker;
     tiny_bits_value value;
     enum tiny_bits_type type = unpack_value(unpacker, &value);
@@ -306,7 +370,7 @@ static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
 
     switch (type) {
         case TINY_BITS_STR: {        
-            return rb_unpack_str(unpacker_data, value, interned);    
+            return rb_unpack_str(strings, unpacker_data, value, interned);    
         }
         case TINY_BITS_DOUBLE:
             return DBL2NUM(value.double_val);
@@ -321,7 +385,7 @@ static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
         case TINY_BITS_ARRAY: {
             VALUE arr = rb_ary_new_capa(value.length);
             for (size_t i = 0; i < value.length; i++) {
-                VALUE element = unpack_ruby_object(unpacker_data, 0);
+                VALUE element = unpack_ruby_object(strings, unpacker_data, 0);
                 if (element == Qundef) return Qundef; // Error
                 rb_ary_push(arr, element);
             }
@@ -330,9 +394,9 @@ static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
         case TINY_BITS_MAP: {
             VALUE hash = rb_hash_new_capa(value.length);
             for (size_t i = 0; i < value.length; i++) {  
-                VALUE key = unpack_ruby_object(unpacker_data, 1);
+                VALUE key = unpack_ruby_object(strings, unpacker_data, 0);
                 if (key == Qundef) return Qundef; // Error
-                VALUE val = unpack_ruby_object(unpacker_data, 0);
+                VALUE val = unpack_ruby_object(strings, unpacker_data, 0);
                 if (val == Qundef) return Qundef; // Error
                 rb_hash_aset(hash, key, val);
             }
@@ -351,6 +415,15 @@ static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
     }
 }
 
+/*
+ * Document-method: unpack
+ *
+ * Unpacks a binary string into a Ruby object.
+ *
+ * @param buffer [String] The binary string to unpack.
+ * @return [Object] The unpacked Ruby object.
+ * @raise [RuntimeError] If unpacking fails.
+ */
 static VALUE rb_unpack(VALUE self, VALUE buffer) {
     UnpackerData* unpacker_data;
     TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
@@ -363,14 +436,27 @@ static VALUE rb_unpack(VALUE self, VALUE buffer) {
 
     tiny_bits_unpacker_set_buffer(unpacker_data->unpacker, (const unsigned char*)RSTRING_PTR(buffer), RSTRING_LEN(buffer));
 
-    VALUE result = unpack_ruby_object(unpacker_data, 0);
+    VALUE array = rb_iv_get(self, "@strings");
+    VALUE result = unpack_ruby_object(array, unpacker_data, 0);
+
     if (result == Qundef) {
         rb_raise(rb_eRuntimeError, "Failed to unpack data");
     }
 
+    rb_ary_clear(array);
+
     return result;
 }
 
+/*
+ * Document-method: buffer=
+ *
+ * Sets the buffer for incremental unpacking.
+ *
+ * @param buffer [String] The binary buffer to unpack incrementally.
+ * @return [Unpacker] self
+ * @note Stores a reference to the buffer internally.
+ */
 static VALUE rb_set_buffer(VALUE self, VALUE buffer){
     UnpackerData* unpacker_data;
     TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
@@ -385,13 +471,25 @@ static VALUE rb_set_buffer(VALUE self, VALUE buffer){
 
     // set the buffer as an instance variable to mainatin a reference to it
     rb_iv_set(self, "@buffer", buffer);
-
+    VALUE array = rb_iv_get(self, "@strings");
+    rb_ary_clear(array);
     return self;
 }
 
+/*
+ * Document-method: pop
+ *
+ * Extracts the next object from the buffer.
+ *
+ * @return [Object, nil] The next unpacked object or nil if buffer is exhausted.
+ * @raise [RuntimeError] If no buffer is set or if unpacking fails.
+ * @note Requires #buffer= to be called first.
+ */
 static VALUE rb_pop(VALUE self) {
 
     VALUE buffer = rb_iv_get(self, "@buffer");
+    VALUE array = rb_iv_get(self, "@strings");
+    
     if(buffer == Qnil){
         rb_raise(rb_eRuntimeError, "No buffer is set");
     }
@@ -404,27 +502,33 @@ static VALUE rb_pop(VALUE self) {
     }
 
     tiny_bits_unpacker* unpacker = unpacker_data->unpacker;
-    tiny_bits_value value;
 
     if(unpacker->current_pos >= unpacker->size - 1){
+        rb_ary_clear(array);
         return Qnil;
     }
-    
-    if(unpacker->current_pos > 0){
-        enum tiny_bits_type type = unpack_value(unpacker, &value);
-        if(type != TINY_BITS_SEP){
-            rb_raise(rb_eRuntimeError, "Malformed multi-object buffer");
-        }
-    }
-    
-    VALUE result = unpack_ruby_object(unpacker_data, 0);
+
+    VALUE result = unpack_ruby_object(array, unpacker_data, 0);
     if (result == Qundef) {
         rb_raise(rb_eRuntimeError, "Failed to unpack data");
     }
 
+    if(unpacker->current_pos >= (unpacker->size - 1)){
+        rb_ary_clear(array);
+    }
+
     return result;
 }
 
+/*
+ * Document-method: finished?
+ *
+ * Checks if the buffer has been fully consumed.
+ *
+ * @return [Boolean] true if there are no more objects to unpack.
+ * @raise [RuntimeError] If no buffer is set.
+ * @note Requires #buffer= to be called first.
+ */
 static VALUE rb_finished(VALUE self){
     VALUE buffer = rb_iv_get(self, "@buffer");
     if(buffer == Qnil){

data/lib/tinybits/dpacker.rb

@@ -0,0 +1,57 @@
+module TinyBits
+ 
+  class DPacker
+    
+    # creates a new TinyBits::DPacker object
+    # @param dict [[String]] an array of strings to use as a dictiontary (order is significant).
+    # @return [TinyBits::DPacker] the dpacker object.
+    # @raise [RuntimeError] the dict is nil or if any member doesn't respnd to to_s.
+    def initialize(dict)
+      @dict = dict.collect{|a| a.to_s }
+      @dict_set = false
+      @packer = Packer.new
+    end
+
+    def reset
+      @packer.reset
+      @dict_set = false
+      self
+    end
+
+    # packs an object to a binary string using the stored dictionary
+    # @param obj [Object] The Ruby object to pack.
+    # @return [String] The packed binary string (frozen).
+    # @raise [RuntimeError] If packing fails due to unsupported types or other errors.
+    def pack(object) 
+      reset
+      self << object
+      to_s
+    end
+
+    # push an object to an inceremental buffer
+    # @param obj [Object] The Ruby object to pack.
+    # @return [Integer] The number of bytes written to the buffer.
+    # @raise [RuntimeError] If packing fails due to unsupported types or other errors.
+    def <<(object)
+      if !@dict_set
+        #@packer.reset
+        @dict_size = @packer << @dict
+        @dict_set = true                
+      end
+      res = @packer << object
+      res
+    end
+
+    # return the packed buffer
+    # @return [String] The packed buffer (minus the dictionary).
+    def to_s
+      res = @packer.to_s
+      if @dict_set
+        res[@dict_size, res.bytesize - @dict_size]
+      else
+        res
+      end
+    end
+
+  end
+end

data/lib/tinybits/dunpacker.rb

@@ -0,0 +1,35 @@
+module TinyBits
+
+  class DUnpacker
+    
+    def initialize(dict)
+      @dict = Packer.new.pack(dict.collect{|a| a.to_s })
+      @unpacker = Unpacker.new
+      @dict_popped = false
+    end
+
+    def unpack(buffer)
+      self.buffer = buffer
+      pop
+    end
+
+    def buffer=(buffer)
+      @unpacker.buffer = "#{@dict}#{buffer}" 
+      @dict_popped = false         
+    end
+
+    def pop
+      if !@dict_popped
+        @unpacker.pop
+        @dict_popped = true              
+      end
+      @unpacker.pop
+    end
+
+    def finished?
+      @unpacker.finished?
+    end
+
+  end
+
+end

data/lib/tinybits/version.rb

@@ -1,3 +1,3 @@
 module TinyBits
-  VERSION = '0.3.0'.freeze
+  VERSION = '0.6.0'.freeze
 end

data/lib/tinybits.rb

@@ -1,7 +1,34 @@
 require_relative './tinybits/version'
 require 'tinybits_ext'
+require_relative './tinybits/dpacker'
+require_relative './tinybits/dunpacker'
 
 module TinyBits
-  def self.pack(object) = Packer.new.pack(object)
-  def self.unpack(buffer) = Unpacker.new.unpack(buffer)
+  # packs an object to a binary string
+  # @param obj [Object] The Ruby object to pack.
+  # @return [String] The packed binary string (frozen).
+  # @raise [RuntimeError] If packing fails due to unsupported types or other errors.
+  # this is a convinience interface, a better way is to instantiate a TinyBits::Packer
+  # object and use its #pack method
+  def self.pack(object, dict = nil)
+    if dict
+      DPacker.new(dict).pack(object)     
+    else
+      Packer.new.pack(object)
+    end
+  end
+
+  # unpacks an object from a binary string
+  # @param buffer [String] The Ruby string holding the packed buffer.
+  # @return [Object] The unpacked Ruby Object (all strings within the object will be frozen).
+  # @raise [RuntimeError] If unpacking fails due to unsupported types or malformed data.
+  # this is a convinience interface, a better way is to instantiate a TinyBits::Unpacker
+  # object and use its #unpack method
+  def self.unpack(buffer, dict = nil)
+    if dict
+      DUnpacker.new(dict).unpack(buffer)
+    else
+      Unpacker.new.unpack(buffer)
+    end
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: tinybits
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Mohamed Hassan
 bindir: bin
 cert_chain: []
-date: 2025-05-04 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: TinyBits is a Ruby gem that wraps the TinyBits C serializartion library,
   offering Rubyists the power of serializion with intger/float compression and string
@@ -19,10 +19,11 @@ extensions:
 extra_rdoc_files: []
 files:
 - ext/tinybits/extconf.rb
-- ext/tinybits/test_date.rb
 - ext/tinybits/tinybits.h
 - ext/tinybits/tinybits_ext.c
 - lib/tinybits.rb
+- lib/tinybits/dpacker.rb
+- lib/tinybits/dunpacker.rb
 - lib/tinybits/version.rb
 homepage: https://github.com/oldmoe/tinybits-rb
 licenses:
@@ -43,7 +44,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.0.dev
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Very fast and compact serialization for Ruby!
 test_files: []

data/ext/tinybits/test_date.rb

@@ -1,62 +0,0 @@
-require './tinybits_ext'
-
-packer = TinyBits::Packer.new
-unpacker = TinyBits::Unpacker.new
-
-t = [Time.now, nil, true, false]
-
-
-
-=begin
-puts packer.dump(t).bytesize
-
-puts t
-puts t2 = unpacker.unpack(packer.pack(t))
-puts t == t2
-=end
-
-class User
-  def initialize(name:, title:)
-    @name = name
-    @title = title
-  end
-  
-  def to_tinybits
-    {"name" => @name, "title" => @title}
-  end
-end
-
-objects = [{"abc": 123}, {"abc": [1, 2, "abc"]}, ["xyz", "abc", "xyz", 7.6] ]
-
-puts "----------------"
-
-#packer.reset
-
-objects.each do |obj|
-  puts packer << obj
-end
-
-buffer = packer.to_s
-
-puts buffer.bytesize
-
-unpacker.buffer = buffer
-
-
-while(value = unpacker.pop)
-  pp value
-  puts "+++++++++++++++++++++++++"
-end
-
-packer.reset
-
-user = User.new(name: "Mohamed", title: "Father")
-
-data = { "user" => user, "tags" => ["user", "Father"] }
-
-pp user
-pp user.to_tinybits
-packed = packer.pack(data)
-puts packed.bytesize
-pp unpacker.unpack(packed)
-