tinybits 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 579aa574466ff4f72f4e9fa59940d3d14fc773404cb73296b2b1498551da6d2f
-  data.tar.gz: 5edf98872ab8d59a3868b0dc6e3b05271d8dbdba7664afc039ca219820a7b247
+  metadata.gz: 9ce29d814129fc925832887a9a4d83efd6dff20a614aab92e2dc80d10197d5aa
+  data.tar.gz: 137f8181f9d0f4c4e3d6d5abe3cb3c0014de07149e08a6e1f4d6c134cbc5c003
 SHA512:
-  metadata.gz: 887b0b5c435cd7da039af2b370f680c5266c06cad1db665e54436f81a22172837495739c051ccea3050a446e055ca24a9198269a51bdb7005fec79d398c37758
-  data.tar.gz: 44c1cd7a40ccf330d8ca96a5c7203b6ce60c07c00c3580fe28440ef0b62c6c083ad4417143a52afce3ae0553eefbc7a9d9035e7011d13557f682704be6f5ad5d
+  metadata.gz: ea777d8a1c1230512e9865e6cababb992f45d70b6c1c8d03d65567c82bda4a514ea1759341decf4c2b5da6934c23d75bc4f01ca7682a32136557382ba368d256
+  data.tar.gz: f16d288249b7cdb434e23d80823a5c5816a7b3be3989f0d0c0ce2513c7255c7f3e1df06b48f1a57681d4ccb3678328f8aa207fcdd5cbe2289cf7372f647aa52a

data/ext/tinybits/tinybits.h

@@ -1,6 +1,6 @@
 /**
  * TinyBits Amalgamated Header
- * Generated on: Sun May  4 05:43:30 PM CEST 2025
+ * Generated on: Tue May 13 11:39:13 PM CEST 2025
  */
 
 #ifndef TINY_BITS_H
@@ -176,46 +176,111 @@ static inline int varint_size(uint64_t value){
     return 9;
 }
 
-static inline uint64_t decode_varint(const uint8_t* buffer, size_t size, size_t *pos) {
+static inline int8_t decode_varint(const uint8_t* buffer, size_t size, size_t pos, uint64_t *value) {
+    if(pos >= size) return 0;
+    uint8_t prefix = buffer[pos];
+    if (prefix <= 240) {
+        *value = prefix;
+        return 1;
+    } else if (prefix >= 241 && prefix <= 248) {
+        if (pos + 1 >= size) return 0; // Not enough bytes
+        *value = 240 + 256 * (prefix - 241) + buffer[pos+1];
+        return 2;
+    } else if (prefix == 249){
+        if (pos + 2 >= size) return 0; // Not enough bytes
+        *value = 2288 + 256 * buffer[pos+1] + buffer[pos+2];
+        return 3;
+    } else if (prefix == 250){
+        if (pos + 3 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 16) | ((uint64_t)buffer[pos+2] << 8) | buffer[pos+3];
+        return 4;
+    } else if (prefix == 251){
+        if (pos + 4 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 24) | ((uint64_t)buffer[pos+2] << 16) |
+               ((uint64_t)buffer[pos+3] << 8) | buffer[pos+4];
+        return 5;    
+    } else if (prefix == 252){
+        if (pos + 5 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 32) | ((uint64_t)buffer[pos+2] << 24) |
+               ((uint64_t)buffer[pos+3] << 16) | ((uint64_t)buffer[pos+4] << 8) | buffer[pos+5];
+        return 6;        
+    } else if (prefix == 253){
+        if (pos + 6 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 40) | ((uint64_t)buffer[pos+2] << 32) |
+               ((uint64_t)buffer[pos+3] << 24) | ((uint64_t)buffer[pos+4] << 16) |
+               ((uint64_t)buffer[pos+5] << 8) | buffer[pos+6];
+        return 7;            
+    } else if (prefix == 254){
+        if (pos + 7 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 48) | ((uint64_t)buffer[pos+2] << 40) |
+               ((uint64_t)buffer[pos+3] << 32) | ((uint64_t)buffer[pos+4] << 24) |
+               ((uint64_t)buffer[pos+5] << 16) | ((uint64_t)buffer[pos+6] << 8) | buffer[pos+7];
+        return 8;            
+    } else if (prefix == 255){
+        if (pos + 8 >= size) return 0; // Not enough bytes
+        *value = ((uint64_t)buffer[pos+1] << 56) | ((uint64_t)buffer[pos+2] << 48) |
+               ((uint64_t)buffer[pos+3] << 40) | ((uint64_t)buffer[pos+4] << 32) |
+               ((uint64_t)buffer[pos+5] << 24) | ((uint64_t)buffer[pos+6] << 16) |
+               ((uint64_t)buffer[pos+7] << 8) | buffer[pos+8];
+        return 9;            
+    } else {
+        return 0;
+    }
+
+}
+
+static inline uint64_t decode_varint_old(const uint8_t* buffer, size_t size, size_t *pos) {
+    if (*pos >= size) {
+        return 0; // not enough buffer
+    }
+
     uint8_t prefix = buffer[*pos];
     if (prefix <= 240) {
         *pos += 1;
         return prefix;
     } else if (prefix >= 241 && prefix <= 248) {
+        if (*pos + 1 >= size) return 0; // Not enough bytes
         uint64_t value = 240 + 256 * (prefix - 241) + buffer[*pos+1];
         *pos += 2;
         return value;
     } else if (prefix == 249) {
+        if (*pos + 2 >= size) return 0; // Not enough bytes
         uint64_t value = 2288 + 256 * buffer[*pos+1] + buffer[*pos+2];
         *pos += 3;
         return value;
     } else if (prefix == 250) {
+        if (*pos + 3 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 16) | ((uint64_t)buffer[*pos+2] << 8) | buffer[*pos+3];
         *pos += 4;
         return value;
     } else if (prefix == 251) {
+        if (*pos + 4 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 24) | ((uint64_t)buffer[*pos+2] << 16) |
                ((uint64_t)buffer[*pos+3] << 8) | buffer[*pos+4];
         *pos += 5;
         return value;
     } else if (prefix == 252) {
+        if (*pos + 5 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 32) | ((uint64_t)buffer[*pos+2] << 24) |
                ((uint64_t)buffer[*pos+3] << 16) | ((uint64_t)buffer[*pos+4] << 8) | buffer[*pos+5];
         *pos += 6;
         return value;
     } else if (prefix == 253) {
+        if (*pos + 6 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 40) | ((uint64_t)buffer[*pos+2] << 32) |
                ((uint64_t)buffer[*pos+3] << 24) | ((uint64_t)buffer[*pos+4] << 16) |
                ((uint64_t)buffer[*pos+5] << 8) | buffer[*pos+6];
         *pos += 7;
         return value;       
     } else if (prefix == 254) {
+        if (*pos + 7 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 48) | ((uint64_t)buffer[*pos+2] << 40) |
                ((uint64_t)buffer[*pos+3] << 32) | ((uint64_t)buffer[*pos+4] << 24) |
                ((uint64_t)buffer[*pos+5] << 16) | ((uint64_t)buffer[*pos+6] << 8) | buffer[*pos+7];
         *pos += 8;
         return value;       
     } else if (prefix == 255) {
+        if (*pos + 8 >= size) return 0; // Not enough bytes
         uint64_t value = ((uint64_t)buffer[*pos+1] << 56) | ((uint64_t)buffer[*pos+2] << 48) |
                ((uint64_t)buffer[*pos+3] << 40) | ((uint64_t)buffer[*pos+4] << 32) |
                ((uint64_t)buffer[*pos+5] << 24) | ((uint64_t)buffer[*pos+6] << 16) |
@@ -399,6 +464,7 @@ tiny_bits_packer *tiny_bits_packer_create(size_t initial_capacity, uint8_t featu
         encoder->encode_table.cache_size = TB_HASH_CACHE_SIZE;
         encoder->encode_table.cache_pos = 0;
         encoder->encode_table.next_id = 0;
+        memset(encoder->encode_table.bins, 0, TB_HASH_SIZE * sizeof(uint8_t));
     } else {
         encoder->encode_table.cache = NULL;
         encoder->encode_table.cache_size = 0;
@@ -953,17 +1019,23 @@ static inline enum tiny_bits_type _unpack_int(tiny_bits_unpacker *decoder, uint8
             value->int_val = tag - 128;
             return TINY_BITS_INT;
         } else if (tag == 248) { // Positive with continuation
-            uint64_t val = decode_varint(decoder->buffer, decoder->size, &pos);
+            uint8_t read;
+            uint64_t val;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &val);
+            if(read == 0) return TINY_BITS_ERROR;
             value->int_val = val + 120;
-            decoder->current_pos = pos;
+            decoder->current_pos += read;
             return TINY_BITS_INT;
         } else if (tag > 248 && tag < 255) { // Small negative (248-254)
             value->int_val = -(tag - 248);
             return TINY_BITS_INT;
         } else { // 255: Negative with continuation
-            uint64_t val = decode_varint(decoder->buffer, decoder->size, &pos);
+            uint8_t read;
+            uint64_t val;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &val);
+            if(read == 0) return TINY_BITS_ERROR;
             value->int_val = -(val + 7);
-            decoder->current_pos = pos;
+            decoder->current_pos += read;
             return TINY_BITS_INT;
         }
 }
@@ -973,8 +1045,12 @@ static inline enum tiny_bits_type _unpack_arr(tiny_bits_unpacker *decoder, uint8
         if (tag < 0b00001111) { // Small array (0-30)
             value->length = tag & 0b00000111;
         } else { // Large array
-            value->length = decode_varint(decoder->buffer, decoder->size, &pos) + 7;
-            decoder->current_pos = pos;
+            uint8_t read;
+            uint64_t val;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &val);
+            if(read == 0) return TINY_BITS_ERROR;
+            value->length = val + 7;
+            decoder->current_pos += read;
         }
         return TINY_BITS_ARRAY;
 }
@@ -984,8 +1060,12 @@ static inline enum tiny_bits_type _unpack_map(tiny_bits_unpacker *decoder, uint8
         if (tag < 0x1F) { // Small map (0-14)
             value->length = tag & 0x0F;
         } else { // Large map
-            value->length = decode_varint(decoder->buffer, decoder->size, &pos) + 15;
-            decoder->current_pos = pos;
+            uint8_t read;
+            uint64_t val;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &val);
+            if(read == 0) return TINY_BITS_ERROR;
+            value->length = val + 15;
+            decoder->current_pos += read;
         }
         return TINY_BITS_MAP;
 }
@@ -993,28 +1073,28 @@ static inline enum tiny_bits_type _unpack_map(tiny_bits_unpacker *decoder, uint8
 static inline enum tiny_bits_type _unpack_double(tiny_bits_unpacker *decoder, uint8_t tag, tiny_bits_value *value){
         size_t pos = decoder->current_pos;
         if (tag == TB_F64_TAG) { // Raw double
+            if(pos + 8 > decoder->size) return TINY_BITS_ERROR;
             uint64_t number = decode_uint64(decoder->buffer + pos);
             value->double_val = itod_bits(number);
             decoder->current_pos += 8;
         } else { // Compressed double
-            uint64_t number = decode_varint(decoder->buffer, decoder->size, &pos);
+            uint8_t read;
+            uint64_t number;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &number);
+            if(read == 0) return TINY_BITS_ERROR;
             int order = (tag & 0x0F); 
             double fractional = (double)number / powers[order];
-            //fractional /= powers[order];
             if(tag & 0x10) fractional = -fractional;        
             value->double_val = fractional;
-            decoder->current_pos = pos;
+            decoder->current_pos += read;
         }
         return TINY_BITS_DOUBLE;
 }
 
 static inline enum tiny_bits_type _unpack_datetime(tiny_bits_unpacker *decoder, uint8_t tag, tiny_bits_value *value){
     size_t pos = decoder->current_pos;
+    if(pos + 8 > decoder->size) return TINY_BITS_ERROR;
     value->datetime_val.offset = decoder->buffer[pos] * (60*15); // convert offset back to seconds (from multiples of 15 minutes)
-    //uint8_t dbl_tag = decoder->buffer[decoder->current_pos++];
-    //tiny_bits_value dbl_val;
-    //_unpack_double(decoder, dbl_tag, &dbl_val);
-    //value->datetime_val.unixtime = dbl_val.double_val;
     uint64_t unixtime = decode_uint64(decoder->buffer + pos + 1);
     value->datetime_val.unixtime = itod_bits(unixtime);
     decoder->current_pos += 9;
@@ -1023,10 +1103,14 @@ static inline enum tiny_bits_type _unpack_datetime(tiny_bits_unpacker *decoder,
 
 static inline enum tiny_bits_type _unpack_blob(tiny_bits_unpacker *decoder, uint8_t tag, tiny_bits_value *value){
         size_t pos = decoder->current_pos;
-        size_t len = decode_varint(decoder->buffer, decoder->size, &pos);
+        size_t len;
+        size_t read; 
+        read = decode_varint(decoder->buffer, decoder->size, pos, &len);
+        if(read == 0) return TINY_BITS_ERROR;
+        if((pos + read + len) > decoder->size) return TINY_BITS_ERROR; 
         value->str_blob_val.data =  (const char *)decoder->buffer + pos;
         value->str_blob_val.length = len; 
-        decoder->current_pos = pos + len;
+        decoder->current_pos = pos + read + len;
         return TINY_BITS_BLOB;
 }
 
@@ -1035,22 +1119,35 @@ static inline enum tiny_bits_type _unpack_str(tiny_bits_unpacker *decoder, uint8
         size_t len;
         if (tag < 0x5F) { // Small string (0-30)
             len = tag & 0x1F;
+            if(pos + len > decoder->size) return TINY_BITS_ERROR;
             value->str_blob_val.data =  (const char *)decoder->buffer + pos;
             value->str_blob_val.length = len; 
-            decoder->current_pos = pos + len;
+            decoder->current_pos += len;
         } else if (tag == 0x5F) { // Large string
-            len = decode_varint(decoder->buffer, decoder->size, &pos) + 31;
+            size_t read;
+            read = decode_varint(decoder->buffer, decoder->size, pos, &len);
+            if(read == 0) return TINY_BITS_ERROR;
+            len += 31;
+            if(pos + read + len > decoder->size) return TINY_BITS_ERROR;
             value->str_blob_val.data =  (const char *)decoder->buffer + pos;
             value->str_blob_val.length = len; 
-            decoder->current_pos = pos + len;
+            decoder->current_pos += (read + len);
         } else { // Deduplicated (small: < 0x7F, large: 0x7F)
-            uint32_t id = (tag < 0x7F) ? (tag & 0x1F) : decode_varint(decoder->buffer, decoder->size, &pos) + 31;
+            size_t id;
+            size_t read;
+            if(tag < 0x7F){
+                id = tag & 0x1F;
+            }else {
+                read = decode_varint(decoder->buffer, decoder->size, pos, &id);
+                if(read == 0) return TINY_BITS_ERROR;
+                id += 31; 
+                decoder->current_pos += read; // Update pos after varint
+            } 
             if (id >= decoder->strings_count) return TINY_BITS_ERROR;
             len = decoder->strings[id].length;
             value->str_blob_val.data = decoder->strings[id].str;
             value->str_blob_val.length = len;
-            value->str_blob_val.id = id+1;
-            decoder->current_pos = pos; // Update pos after varint
+            value->str_blob_val.id = id + 1;
             return TINY_BITS_STR;
         }
         value->str_blob_val.id = 0;

data/ext/tinybits/tinybits_ext.c

@@ -225,13 +225,13 @@ static VALUE rb_push(VALUE self, VALUE obj) {
     context.packer = packer_data->packer;        // Pass the current packer
     context.error_occurred = 0;   // Initialize error flag
 
-    
+    /*
     if(initial_pos > 0){
        if(!pack_separator(packer_data->packer)){
             rb_raise(rb_eRuntimeError, "Failed to pack object (multi-object packing error)");
        } 
     }
-    
+    */
     // Call the optimized recursive function
     if (!pack_ruby_object_recursive(packer_data->packer, obj, (VALUE)&context)) {
         // Error occurred during packing (might be unsupported type or tiny_bits error)
@@ -264,6 +264,13 @@ static VALUE rb_to_s(VALUE self){
     return result;
 }
 
+/*
+ * Document-method: reset
+ *
+ * Resets the packer's buffer to empty.
+ *
+ * @return [Packer] self
+ */
 static VALUE rb_reset(VALUE self){
     PackerData* packer_data;
     TypedData_Get_Struct(self, PackerData, &packer_data_type, packer_data);
@@ -281,8 +288,8 @@ static VALUE rb_reset(VALUE self){
 // Unpacker structure
 typedef struct {
     tiny_bits_unpacker* unpacker;
-    size_t strings_index;
-    VALUE ruby_strings[TB_HASH_CACHE_SIZE];
+    //size_t strings_index;
+    //VALUE ruby_strings[TB_HASH_CACHE_SIZE];
 } UnpackerData;
 
 static void unpacker_free(void* data) {
@@ -290,9 +297,9 @@ static void unpacker_free(void* data) {
     if (unpacker_data->unpacker) {
         tiny_bits_unpacker_destroy(unpacker_data->unpacker);
     }
-    for (size_t i = 0; i < TB_HASH_CACHE_SIZE; i++) {
-        unpacker_data->ruby_strings[i] = Qnil;
-    }
+    //for (size_t i = 0; i < TB_HASH_CACHE_SIZE; i++) {
+        //unpacker_data->ruby_strings[i] = Qnil;
+    //}
     free(unpacker_data);
 }
 
@@ -312,6 +319,13 @@ static VALUE rb_unpacker_alloc(VALUE klass) {
     return TypedData_Wrap_Struct(klass, &unpacker_data_type, unpacker_data);
 }
 
+/*
+ * Document-method: initialize
+ *
+ * Initializes a new Unpacker.
+ *
+ * @return [Unpacker] The initialized unpacker object.
+ */
 static VALUE rb_unpacker_init(VALUE self) {
     UnpackerData* unpacker_data;
     TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
@@ -320,14 +334,15 @@ static VALUE rb_unpacker_init(VALUE self) {
     if (!unpacker_data->unpacker) {
         rb_raise(rb_eRuntimeError, "Failed to initialize unpacker");
     }
-    unpacker_data->strings_index = 0;
+    VALUE strings = rb_ary_new_capa(TB_HASH_CACHE_SIZE);
+    rb_iv_set(self, "@strings", strings);
     return self;
 }
 
-static inline VALUE rb_unpack_str(UnpackerData* unpacker_data, tiny_bits_value value, size_t interned){
+static inline VALUE rb_unpack_str(VALUE strings, UnpackerData* unpacker_data, tiny_bits_value value, size_t interned){
     int32_t id  = value.str_blob_val.id;
     if(id > 0)
-        return unpacker_data->ruby_strings[id-1];
+        return rb_ary_entry(strings, id-1);
     else if(id <= 0){
         VALUE str;
         if(interned > 0){
@@ -337,14 +352,14 @@ static inline VALUE rb_unpack_str(UnpackerData* unpacker_data, tiny_bits_value v
             rb_obj_freeze(str);
         }
         if(id < 0){
-            unpacker_data->ruby_strings[abs(id)-1] = str;
+            rb_ary_push(strings, str);
         }
         return str;
     }            
     return Qundef;
 }
 
-static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
+static VALUE unpack_ruby_object(VALUE strings, UnpackerData* unpacker_data, size_t interned) {
     tiny_bits_unpacker* unpacker = unpacker_data->unpacker;
     tiny_bits_value value;
     enum tiny_bits_type type = unpack_value(unpacker, &value);
@@ -355,7 +370,7 @@ static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
 
     switch (type) {
         case TINY_BITS_STR: {        
-            return rb_unpack_str(unpacker_data, value, interned);    
+            return rb_unpack_str(strings, unpacker_data, value, interned);    
         }
         case TINY_BITS_DOUBLE:
             return DBL2NUM(value.double_val);
@@ -370,7 +385,7 @@ static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
         case TINY_BITS_ARRAY: {
             VALUE arr = rb_ary_new_capa(value.length);
             for (size_t i = 0; i < value.length; i++) {
-                VALUE element = unpack_ruby_object(unpacker_data, 0);
+                VALUE element = unpack_ruby_object(strings, unpacker_data, 0);
                 if (element == Qundef) return Qundef; // Error
                 rb_ary_push(arr, element);
             }
@@ -379,9 +394,9 @@ static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
         case TINY_BITS_MAP: {
             VALUE hash = rb_hash_new_capa(value.length);
             for (size_t i = 0; i < value.length; i++) {  
-                VALUE key = unpack_ruby_object(unpacker_data, 1);
+                VALUE key = unpack_ruby_object(strings, unpacker_data, 0);
                 if (key == Qundef) return Qundef; // Error
-                VALUE val = unpack_ruby_object(unpacker_data, 0);
+                VALUE val = unpack_ruby_object(strings, unpacker_data, 0);
                 if (val == Qundef) return Qundef; // Error
                 rb_hash_aset(hash, key, val);
             }
@@ -400,6 +415,15 @@ static VALUE unpack_ruby_object(UnpackerData* unpacker_data, size_t interned) {
     }
 }
 
+/*
+ * Document-method: unpack
+ *
+ * Unpacks a binary string into a Ruby object.
+ *
+ * @param buffer [String] The binary string to unpack.
+ * @return [Object] The unpacked Ruby object.
+ * @raise [RuntimeError] If unpacking fails.
+ */
 static VALUE rb_unpack(VALUE self, VALUE buffer) {
     UnpackerData* unpacker_data;
     TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
@@ -412,14 +436,27 @@ static VALUE rb_unpack(VALUE self, VALUE buffer) {
 
     tiny_bits_unpacker_set_buffer(unpacker_data->unpacker, (const unsigned char*)RSTRING_PTR(buffer), RSTRING_LEN(buffer));
 
-    VALUE result = unpack_ruby_object(unpacker_data, 0);
+    VALUE array = rb_iv_get(self, "@strings");
+    VALUE result = unpack_ruby_object(array, unpacker_data, 0);
+
     if (result == Qundef) {
         rb_raise(rb_eRuntimeError, "Failed to unpack data");
     }
 
+    rb_ary_clear(array);
+
     return result;
 }
 
+/*
+ * Document-method: buffer=
+ *
+ * Sets the buffer for incremental unpacking.
+ *
+ * @param buffer [String] The binary buffer to unpack incrementally.
+ * @return [Unpacker] self
+ * @note Stores a reference to the buffer internally.
+ */
 static VALUE rb_set_buffer(VALUE self, VALUE buffer){
     UnpackerData* unpacker_data;
     TypedData_Get_Struct(self, UnpackerData, &unpacker_data_type, unpacker_data);
@@ -434,13 +471,25 @@ static VALUE rb_set_buffer(VALUE self, VALUE buffer){
 
     // set the buffer as an instance variable to mainatin a reference to it
     rb_iv_set(self, "@buffer", buffer);
-
+    VALUE array = rb_iv_get(self, "@strings");
+    rb_ary_clear(array);
     return self;
 }
 
+/*
+ * Document-method: pop
+ *
+ * Extracts the next object from the buffer.
+ *
+ * @return [Object, nil] The next unpacked object or nil if buffer is exhausted.
+ * @raise [RuntimeError] If no buffer is set or if unpacking fails.
+ * @note Requires #buffer= to be called first.
+ */
 static VALUE rb_pop(VALUE self) {
 
     VALUE buffer = rb_iv_get(self, "@buffer");
+    VALUE array = rb_iv_get(self, "@strings");
+    
     if(buffer == Qnil){
         rb_raise(rb_eRuntimeError, "No buffer is set");
     }
@@ -453,27 +502,33 @@ static VALUE rb_pop(VALUE self) {
     }
 
     tiny_bits_unpacker* unpacker = unpacker_data->unpacker;
-    tiny_bits_value value;
 
     if(unpacker->current_pos >= unpacker->size - 1){
+        rb_ary_clear(array);
         return Qnil;
     }
-    
-    if(unpacker->current_pos > 0){
-        enum tiny_bits_type type = unpack_value(unpacker, &value);
-        if(type != TINY_BITS_SEP){
-            rb_raise(rb_eRuntimeError, "Malformed multi-object buffer");
-        }
-    }
-    
-    VALUE result = unpack_ruby_object(unpacker_data, 0);
+
+    VALUE result = unpack_ruby_object(array, unpacker_data, 0);
     if (result == Qundef) {
         rb_raise(rb_eRuntimeError, "Failed to unpack data");
     }
 
+    if(unpacker->current_pos >= (unpacker->size - 1)){
+        rb_ary_clear(array);
+    }
+
     return result;
 }
 
+/*
+ * Document-method: finished?
+ *
+ * Checks if the buffer has been fully consumed.
+ *
+ * @return [Boolean] true if there are no more objects to unpack.
+ * @raise [RuntimeError] If no buffer is set.
+ * @note Requires #buffer= to be called first.
+ */
 static VALUE rb_finished(VALUE self){
     VALUE buffer = rb_iv_get(self, "@buffer");
     if(buffer == Qnil){

data/lib/tinybits/dpacker.rb

@@ -0,0 +1,57 @@
+module TinyBits
+ 
+  class DPacker
+    
+    # creates a new TinyBits::DPacker object
+    # @param dict [[String]] an array of strings to use as a dictiontary (order is significant).
+    # @return [TinyBits::DPacker] the dpacker object.
+    # @raise [RuntimeError] the dict is nil or if any member doesn't respnd to to_s.
+    def initialize(dict)
+      @dict = dict.collect{|a| a.to_s }
+      @dict_set = false
+      @packer = Packer.new
+    end
+
+    def reset
+      @packer.reset
+      @dict_set = false
+      self
+    end
+
+    # packs an object to a binary string using the stored dictionary
+    # @param obj [Object] The Ruby object to pack.
+    # @return [String] The packed binary string (frozen).
+    # @raise [RuntimeError] If packing fails due to unsupported types or other errors.
+    def pack(object) 
+      reset
+      self << object
+      to_s
+    end
+
+    # push an object to an inceremental buffer
+    # @param obj [Object] The Ruby object to pack.
+    # @return [Integer] The number of bytes written to the buffer.
+    # @raise [RuntimeError] If packing fails due to unsupported types or other errors.
+    def <<(object)
+      if !@dict_set
+        #@packer.reset
+        @dict_size = @packer << @dict
+        @dict_set = true                
+      end
+      res = @packer << object
+      res
+    end
+
+    # return the packed buffer
+    # @return [String] The packed buffer (minus the dictionary).
+    def to_s
+      res = @packer.to_s
+      if @dict_set
+        res[@dict_size, res.bytesize - @dict_size]
+      else
+        res
+      end
+    end
+
+  end
+end

data/lib/tinybits/dunpacker.rb

@@ -0,0 +1,35 @@
+module TinyBits
+
+  class DUnpacker
+    
+    def initialize(dict)
+      @dict = Packer.new.pack(dict.collect{|a| a.to_s })
+      @unpacker = Unpacker.new
+      @dict_popped = false
+    end
+
+    def unpack(buffer)
+      self.buffer = buffer
+      pop
+    end
+
+    def buffer=(buffer)
+      @unpacker.buffer = "#{@dict}#{buffer}" 
+      @dict_popped = false         
+    end
+
+    def pop
+      if !@dict_popped
+        @unpacker.pop
+        @dict_popped = true              
+      end
+      @unpacker.pop
+    end
+
+    def finished?
+      @unpacker.finished?
+    end
+
+  end
+
+end

data/lib/tinybits/version.rb

@@ -1,3 +1,3 @@
 module TinyBits
-  VERSION = '0.4.0'.freeze
+  VERSION = '0.6.0'.freeze
 end

data/lib/tinybits.rb

@@ -1,5 +1,7 @@
 require_relative './tinybits/version'
 require 'tinybits_ext'
+require_relative './tinybits/dpacker'
+require_relative './tinybits/dunpacker'
 
 module TinyBits
   # packs an object to a binary string
@@ -8,7 +10,13 @@ module TinyBits
   # @raise [RuntimeError] If packing fails due to unsupported types or other errors.
   # this is a convinience interface, a better way is to instantiate a TinyBits::Packer
   # object and use its #pack method
-  def self.pack(object) = Packer.new.pack(object)
+  def self.pack(object, dict = nil)
+    if dict
+      DPacker.new(dict).pack(object)     
+    else
+      Packer.new.pack(object)
+    end
+  end
 
   # unpacks an object from a binary string
   # @param buffer [String] The Ruby string holding the packed buffer.
@@ -16,5 +24,11 @@ module TinyBits
   # @raise [RuntimeError] If unpacking fails due to unsupported types or malformed data.
   # this is a convinience interface, a better way is to instantiate a TinyBits::Unpacker
   # object and use its #unpack method
-  def self.unpack(buffer) = Unpacker.new.unpack(buffer)
+  def self.unpack(buffer, dict = nil)
+    if dict
+      DUnpacker.new(dict).unpack(buffer)
+    else
+      Unpacker.new.unpack(buffer)
+    end
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: tinybits
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Mohamed Hassan
 bindir: bin
 cert_chain: []
-date: 2025-05-04 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: TinyBits is a Ruby gem that wraps the TinyBits C serializartion library,
   offering Rubyists the power of serializion with intger/float compression and string
@@ -19,10 +19,11 @@ extensions:
 extra_rdoc_files: []
 files:
 - ext/tinybits/extconf.rb
-- ext/tinybits/test_date.rb
 - ext/tinybits/tinybits.h
 - ext/tinybits/tinybits_ext.c
 - lib/tinybits.rb
+- lib/tinybits/dpacker.rb
+- lib/tinybits/dunpacker.rb
 - lib/tinybits/version.rb
 homepage: https://github.com/oldmoe/tinybits-rb
 licenses:
@@ -43,7 +44,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.0.dev
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Very fast and compact serialization for Ruby!
 test_files: []

data/ext/tinybits/test_date.rb

@@ -1,62 +0,0 @@
-require './tinybits_ext'
-
-packer = TinyBits::Packer.new
-unpacker = TinyBits::Unpacker.new
-
-t = [Time.now, nil, true, false]
-
-
-
-=begin
-puts packer.dump(t).bytesize
-
-puts t
-puts t2 = unpacker.unpack(packer.pack(t))
-puts t == t2
-=end
-
-class User
-  def initialize(name:, title:)
-    @name = name
-    @title = title
-  end
-  
-  def to_tinybits
-    {"name" => @name, "title" => @title}
-  end
-end
-
-objects = [{"abc": 123}, {"abc": [1, 2, "abc"]}, ["xyz", "abc", "xyz", 7.6] ]
-
-puts "----------------"
-
-#packer.reset
-
-objects.each do |obj|
-  puts packer << obj
-end
-
-buffer = packer.to_s
-
-puts buffer.bytesize
-
-unpacker.buffer = buffer
-
-
-while(value = unpacker.pop)
-  pp value
-  puts "+++++++++++++++++++++++++"
-end
-
-packer.reset
-
-user = User.new(name: "Mohamed", title: "Father")
-
-data = { "user" => user, "tags" => ["user", "Father"] }
-
-pp user
-pp user.to_tinybits
-packed = packer.pack(data)
-puts packed.bytesize
-pp unpacker.unpack(packed)
-