msgpack 1.3.0 → 1.6.0

data/ext/msgpack/factory_class.c

@@ -30,47 +30,76 @@ typedef struct msgpack_factory_t msgpack_factory_t;
 
 struct msgpack_factory_t {
     msgpack_packer_ext_registry_t pkrg;
-    msgpack_unpacker_ext_registry_t ukrg;
+    msgpack_unpacker_ext_registry_t *ukrg;
+    bool has_bigint_ext_type;
     bool has_symbol_ext_type;
+    bool optimized_symbol_ext_type;
+    int symbol_ext_type;
 };
 
-#define FACTORY(from, name) \
-    msgpack_factory_t *name = NULL; \
-    Data_Get_Struct(from, msgpack_factory_t, name); \
-    if(name == NULL) { \
-        rb_raise(rb_eArgError, "NULL found for " # name " when shouldn't be."); \
-    }
-
-static void Factory_free(msgpack_factory_t* fc)
+static void Factory_free(void *ptr)
 {
+    msgpack_factory_t *fc = ptr;
+
     if(fc == NULL) {
         return;
     }
     msgpack_packer_ext_registry_destroy(&fc->pkrg);
-    msgpack_unpacker_ext_registry_destroy(&fc->ukrg);
+    msgpack_unpacker_ext_registry_release(fc->ukrg);
     xfree(fc);
 }
 
-void Factory_mark(msgpack_factory_t* fc)
+void Factory_mark(void *ptr)
 {
+    msgpack_factory_t *fc = ptr;
     msgpack_packer_ext_registry_mark(&fc->pkrg);
-    msgpack_unpacker_ext_registry_mark(&fc->ukrg);
+    msgpack_unpacker_ext_registry_mark(fc->ukrg);
 }
 
-static VALUE Factory_alloc(VALUE klass)
+static size_t Factory_memsize(const void *ptr)
 {
-    msgpack_factory_t* fc = ZALLOC_N(msgpack_factory_t, 1);
+    const msgpack_factory_t *fc = ptr;
+    size_t total_size = sizeof(msgpack_factory_t);
 
-    VALUE self = Data_Wrap_Struct(klass, Factory_mark, Factory_free, fc);
-    return self;
+    if (fc->ukrg) {
+        total_size += sizeof(msgpack_unpacker_ext_registry_t) / (fc->ukrg->borrow_count + 1);
+    }
+
+    return total_size;
+}
+
+static const rb_data_type_t factory_data_type = {
+    .wrap_struct_name = "msgpack:factory",
+    .function = {
+        .dmark = Factory_mark,
+        .dfree = Factory_free,
+        .dsize = Factory_memsize,
+    },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static inline msgpack_factory_t *Factory_get(VALUE object)
+{
+    msgpack_factory_t *factory;
+    TypedData_Get_Struct(object, msgpack_factory_t, &factory_data_type, factory);
+    if (!factory) {
+        rb_raise(rb_eArgError, "Uninitialized Factory object");
+    }
+    return factory;
+}
+
+static VALUE Factory_alloc(VALUE klass)
+{
+    msgpack_factory_t *fc;
+    return TypedData_Make_Struct(klass, msgpack_factory_t, &factory_data_type, fc);
 }
 
 static VALUE Factory_initialize(int argc, VALUE* argv, VALUE self)
 {
-    FACTORY(self, fc);
+    msgpack_factory_t *fc = Factory_get(self);
 
     msgpack_packer_ext_registry_init(&fc->pkrg);
-    msgpack_unpacker_ext_registry_init(&fc->ukrg);
+    // fc->ukrg is lazily initialized
 
     fc->has_symbol_ext_type = false;
 
@@ -85,18 +114,52 @@ static VALUE Factory_initialize(int argc, VALUE* argv, VALUE self)
     return Qnil;
 }
 
+static VALUE Factory_dup(VALUE self)
+{
+    VALUE clone = Factory_alloc(rb_obj_class(self));
+
+    msgpack_factory_t *fc = Factory_get(self);
+    msgpack_factory_t *cloned_fc = Factory_get(clone);
+
+    cloned_fc->has_symbol_ext_type = fc->has_symbol_ext_type;
+    cloned_fc->pkrg = fc->pkrg;
+    msgpack_unpacker_ext_registry_borrow(fc->ukrg, &cloned_fc->ukrg);
+    msgpack_packer_ext_registry_dup(&fc->pkrg, &cloned_fc->pkrg);
+
+    return clone;
+}
+
+static VALUE Factory_freeze(VALUE self) {
+    if(!rb_obj_frozen_p(self)) {
+        msgpack_factory_t *fc = Factory_get(self);
+
+        if (RTEST(fc->pkrg.hash)) {
+            rb_hash_freeze(fc->pkrg.hash);
+            if (!RTEST(fc->pkrg.cache)) {
+                // If the factory is frozen, we can safely share the packer cache between
+                // all packers. So we eagerly create it now so it's available when #packer
+                // is called.
+                fc->pkrg.cache = rb_hash_new();
+            }
+        }
+
+        rb_obj_freeze(self);
+    }
+
+    return self;
+}
+
 VALUE MessagePack_Factory_packer(int argc, VALUE* argv, VALUE self)
 {
-    FACTORY(self, fc);
+    msgpack_factory_t *fc = Factory_get(self);
 
     VALUE packer = MessagePack_Packer_alloc(cMessagePack_Packer);
     MessagePack_Packer_initialize(argc, argv, packer);
 
-    msgpack_packer_t* pk;
-    Data_Get_Struct(packer, msgpack_packer_t, pk);
-
+    msgpack_packer_t* pk = MessagePack_Packer_get(packer);
     msgpack_packer_ext_registry_destroy(&pk->ext_registry);
     msgpack_packer_ext_registry_dup(&fc->pkrg, &pk->ext_registry);
+    pk->has_bigint_ext_type = fc->has_bigint_ext_type;
     pk->has_symbol_ext_type = fc->has_symbol_ext_type;
 
     return packer;
@@ -104,44 +167,47 @@ VALUE MessagePack_Factory_packer(int argc, VALUE* argv, VALUE self)
 
 VALUE MessagePack_Factory_unpacker(int argc, VALUE* argv, VALUE self)
 {
-    FACTORY(self, fc);
+    msgpack_factory_t *fc = Factory_get(self);
 
     VALUE unpacker = MessagePack_Unpacker_alloc(cMessagePack_Unpacker);
     MessagePack_Unpacker_initialize(argc, argv, unpacker);
 
-    msgpack_unpacker_t* uk;
-    Data_Get_Struct(unpacker, msgpack_unpacker_t, uk);
-
-    msgpack_unpacker_ext_registry_destroy(&uk->ext_registry);
-    msgpack_unpacker_ext_registry_dup(&fc->ukrg, &uk->ext_registry);
+    msgpack_unpacker_t* uk = MessagePack_Unpacker_get(unpacker);
+    msgpack_unpacker_ext_registry_borrow(fc->ukrg, &uk->ext_registry);
+    uk->optimized_symbol_ext_type = fc->optimized_symbol_ext_type;
+    uk->symbol_ext_type = fc->symbol_ext_type;
 
     return unpacker;
 }
 
 static VALUE Factory_registered_types_internal(VALUE self)
 {
-    FACTORY(self, fc);
+    msgpack_factory_t *fc = Factory_get(self);
 
     VALUE uk_mapping = rb_hash_new();
-    for(int i=0; i < 256; i++) {
-        if(fc->ukrg.array[i] != Qnil) {
-            rb_hash_aset(uk_mapping, INT2FIX(i - 128), fc->ukrg.array[i]);
+    if (fc->ukrg) {
+        for(int i=0; i < 256; i++) {
+            if(fc->ukrg->array[i] != Qnil) {
+                rb_hash_aset(uk_mapping, INT2FIX(i - 128), fc->ukrg->array[i]);
+            }
         }
     }
-#ifdef HAVE_RB_HASH_DUP
-    return rb_ary_new3(2, rb_hash_dup(fc->pkrg.hash), uk_mapping);
-#else
-    return rb_ary_new3(2, rb_funcall(fc->pkrg.hash, rb_intern("dup"), 0), uk_mapping);
-#endif
+
+    return rb_ary_new3(
+        2,
+        RTEST(fc->pkrg.hash) ? rb_hash_dup(fc->pkrg.hash) : rb_hash_new(),
+        uk_mapping
+    );
 }
 
 static VALUE Factory_register_type(int argc, VALUE* argv, VALUE self)
 {
-    FACTORY(self, fc);
+    msgpack_factory_t *fc = Factory_get(self);
 
     int ext_type;
+    int flags = 0;
     VALUE ext_module;
-    VALUE options;
+    VALUE options = Qnil;
     VALUE packer_arg, unpacker_arg;
     VALUE packer_proc, unpacker_proc;
 
@@ -168,6 +234,10 @@ static VALUE Factory_register_type(int argc, VALUE* argv, VALUE self)
         rb_raise(rb_eArgError, "wrong number of arguments (%d for 2..3)", argc);
     }
 
+    if (options != Qnil) {
+        Check_Type(options, T_HASH);
+    }
+
     ext_type = NUM2INT(argv[0]);
     if(ext_type < -128 || ext_type > 127) {
         rb_raise(rb_eRangeError, "integer %d too big to convert to `signed char'", ext_type);
@@ -188,18 +258,36 @@ static VALUE Factory_register_type(int argc, VALUE* argv, VALUE self)
     if(unpacker_arg != Qnil) {
         if(rb_type(unpacker_arg) == T_SYMBOL || rb_type(unpacker_arg) == T_STRING) {
             unpacker_proc = rb_obj_method(ext_module, unpacker_arg);
+        } else if (rb_respond_to(unpacker_arg, rb_intern("call"))) {
+            unpacker_proc = unpacker_arg;
         } else {
             unpacker_proc = rb_funcall(unpacker_arg, rb_intern("method"), 1, ID2SYM(rb_intern("call")));
         }
     }
 
-    msgpack_packer_ext_registry_put(&fc->pkrg, ext_module, ext_type, packer_proc, packer_arg);
-
-    if (ext_module == rb_cSymbol) {
+    if(ext_module == rb_cSymbol) {
         fc->has_symbol_ext_type = true;
+        if(RTEST(options) && RTEST(rb_hash_aref(options, ID2SYM(rb_intern("optimized_symbols_parsing"))))) {
+            fc->optimized_symbol_ext_type = true;
+        }
+    }
+
+    if(RTEST(options)) {
+        if(RTEST(rb_hash_aref(options, ID2SYM(rb_intern("oversized_integer_extension"))))) {
+            if(ext_module == rb_cInteger) {
+                fc->has_bigint_ext_type = true;
+            } else {
+                rb_raise(rb_eArgError, "oversized_integer_extension: true is only for Integer class");
+            }
+        }
+
+        if(RTEST(rb_hash_aref(options, ID2SYM(rb_intern("recursive"))))) {
+            flags |= MSGPACK_EXT_RECURSIVE;
+        }
     }
 
-    msgpack_unpacker_ext_registry_put(&fc->ukrg, ext_module, ext_type, unpacker_proc, unpacker_arg);
+    msgpack_packer_ext_registry_put(&fc->pkrg, ext_module, ext_type, flags, packer_proc, packer_arg);
+    msgpack_unpacker_ext_registry_put(&fc->ukrg, ext_module, ext_type, flags, unpacker_proc, unpacker_arg);
 
     return Qnil;
 }
@@ -211,6 +299,8 @@ void MessagePack_Factory_module_init(VALUE mMessagePack)
     rb_define_alloc_func(cMessagePack_Factory, Factory_alloc);
 
     rb_define_method(cMessagePack_Factory, "initialize", Factory_initialize, -1);
+    rb_define_method(cMessagePack_Factory, "dup", Factory_dup, 0);
+    rb_define_method(cMessagePack_Factory, "freeze", Factory_freeze, 0);
 
     rb_define_method(cMessagePack_Factory, "packer", MessagePack_Factory_packer, -1);
     rb_define_method(cMessagePack_Factory, "unpacker", MessagePack_Factory_unpacker, -1);

data/ext/msgpack/packer.c

@@ -18,34 +18,18 @@
 
 #include "packer.h"
 
-#ifdef RUBINIUS
-static ID s_to_iter;
-static ID s_next;
-static ID s_key;
-static ID s_value;
-#endif
-
 static ID s_call;
 
-void msgpack_packer_static_init()
+void msgpack_packer_static_init(void)
 {
-#ifdef RUBINIUS
-    s_to_iter = rb_intern("to_iter");
-    s_next = rb_intern("next");
-    s_key = rb_intern("key");
-    s_value = rb_intern("value");
-#endif
-
     s_call = rb_intern("call");
 }
 
-void msgpack_packer_static_destroy()
+void msgpack_packer_static_destroy(void)
 { }
 
 void msgpack_packer_init(msgpack_packer_t* pk)
 {
-    memset(pk, 0, sizeof(msgpack_packer_t));
-
     msgpack_buffer_init(PACKER_BUFFER_(pk));
 }
 
@@ -108,30 +92,64 @@ void msgpack_packer_write_hash_value(msgpack_packer_t* pk, VALUE v)
     unsigned int len32 = (unsigned int)len;
     msgpack_packer_write_map_header(pk, len32);
 
-#ifdef RUBINIUS
-    VALUE iter = rb_funcall(v, s_to_iter, 0);
-    VALUE entry = Qnil;
-    while(RTEST(entry = rb_funcall(iter, s_next, 1, entry))) {
-        VALUE key = rb_funcall(entry, s_key, 0);
-        VALUE val = rb_funcall(entry, s_value, 0);
-        write_hash_foreach(key, val, (VALUE) pk);
-    }
-#else
     rb_hash_foreach(v, write_hash_foreach, (VALUE) pk);
-#endif
 }
 
-void msgpack_packer_write_other_value(msgpack_packer_t* pk, VALUE v)
+struct msgpack_call_proc_args_t;
+typedef struct msgpack_call_proc_args_t msgpack_call_proc_args_t;
+struct msgpack_call_proc_args_t {
+    VALUE proc;
+    VALUE arg;
+    VALUE packer;
+};
+
+VALUE msgpack_packer_try_calling_proc(VALUE value)
 {
-    int ext_type;
+    msgpack_call_proc_args_t *args = (msgpack_call_proc_args_t *)value;
+    return rb_funcall(args->proc, s_call, 2, args->arg, args->packer);
+}
 
-    VALUE proc = msgpack_packer_ext_registry_lookup(&pk->ext_registry, v, &ext_type);
+bool msgpack_packer_try_write_with_ext_type_lookup(msgpack_packer_t* pk, VALUE v)
+{
+    int ext_type, ext_flags;
 
-    if(proc != Qnil) {
+    VALUE proc = msgpack_packer_ext_registry_lookup(&pk->ext_registry, v, &ext_type, &ext_flags);
+
+    if(proc == Qnil) {
+        return false;
+    }
+
+    if(ext_flags & MSGPACK_EXT_RECURSIVE) {
+        msgpack_buffer_t parent_buffer = pk->buffer;
+        msgpack_buffer_init(PACKER_BUFFER_(pk));
+
+        int exception_occured = 0;
+        msgpack_call_proc_args_t args = { proc, v, pk->to_msgpack_arg };
+        rb_protect(msgpack_packer_try_calling_proc, (VALUE)&args, &exception_occured);
+
+        if (exception_occured) {
+            msgpack_buffer_destroy(PACKER_BUFFER_(pk));
+            pk->buffer = parent_buffer;
+            rb_jump_tag(exception_occured); // re-raise the exception
+        } else {
+            VALUE payload = msgpack_buffer_all_as_string(PACKER_BUFFER_(pk));
+            StringValue(payload);
+            msgpack_buffer_destroy(PACKER_BUFFER_(pk));
+            pk->buffer = parent_buffer;
+            msgpack_packer_write_ext(pk, ext_type, payload);
+        }
+    } else {
         VALUE payload = rb_funcall(proc, s_call, 1, v);
         StringValue(payload);
         msgpack_packer_write_ext(pk, ext_type, payload);
-    } else {
+    }
+
+    return true;
+}
+
+void msgpack_packer_write_other_value(msgpack_packer_t* pk, VALUE v)
+{
+    if(!(msgpack_packer_try_write_with_ext_type_lookup(pk, v))) {
         rb_funcall(v, pk->to_msgpack_method, 1, pk->to_msgpack_arg);
     }
 }
@@ -155,13 +173,19 @@ void msgpack_packer_write_value(msgpack_packer_t* pk, VALUE v)
         msgpack_packer_write_symbol_value(pk, v);
         break;
     case T_STRING:
-        msgpack_packer_write_string_value(pk, v);
+        if(rb_class_of(v) == rb_cString || !msgpack_packer_try_write_with_ext_type_lookup(pk, v)) {
+            msgpack_packer_write_string_value(pk, v);
+        }
         break;
     case T_ARRAY:
-        msgpack_packer_write_array_value(pk, v);
+        if(rb_class_of(v) == rb_cArray || !msgpack_packer_try_write_with_ext_type_lookup(pk, v)) {
+            msgpack_packer_write_array_value(pk, v);
+        }
         break;
     case T_HASH:
-        msgpack_packer_write_hash_value(pk, v);
+        if(rb_class_of(v) == rb_cHash || !msgpack_packer_try_write_with_ext_type_lookup(pk, v)) {
+            msgpack_packer_write_hash_value(pk, v);
+        }
         break;
     case T_BIGNUM:
         msgpack_packer_write_bignum_value(pk, v);

data/ext/msgpack/packer.h

@@ -32,6 +32,7 @@ struct msgpack_packer_t {
     msgpack_buffer_t buffer;
 
     bool compatibility_mode;
+    bool has_bigint_ext_type;
     bool has_symbol_ext_type;
 
     ID to_msgpack_method;
@@ -46,9 +47,9 @@ struct msgpack_packer_t {
 
 #define PACKER_BUFFER_(pk) (&(pk)->buffer)
 
-void msgpack_packer_static_init();
+void msgpack_packer_static_init(void);
 
-void msgpack_packer_static_destroy();
+void msgpack_packer_static_destroy(void);
 
 void msgpack_packer_init(msgpack_packer_t* pk);
 
@@ -56,6 +57,8 @@ void msgpack_packer_destroy(msgpack_packer_t* pk);
 
 void msgpack_packer_mark(msgpack_packer_t* pk);
 
+bool msgpack_packer_try_write_with_ext_type_lookup(msgpack_packer_t* pk, VALUE v);
+
 static inline void msgpack_packer_set_to_msgpack_method(msgpack_packer_t* pk,
         ID to_msgpack_method, VALUE to_msgpack_arg)
 {
@@ -396,7 +399,6 @@ static inline void msgpack_packer_write_ext(msgpack_packer_t* pk, int ext_type,
     msgpack_buffer_append_string(PACKER_BUFFER_(pk), payload);
 }
 
-#ifdef COMPAT_HAVE_ENCODING
 static inline bool msgpack_packer_is_binary(VALUE v, int encindex)
 {
     return encindex == msgpack_rb_encindex_ascii8bit;
@@ -406,15 +408,8 @@ static inline bool msgpack_packer_is_utf8_compat_string(VALUE v, int encindex)
 {
     return encindex == msgpack_rb_encindex_utf8
         || encindex == msgpack_rb_encindex_usascii
-#ifdef ENC_CODERANGE_ASCIIONLY
-        /* Because ENC_CODERANGE_ASCIIONLY does not scan string, it may return ENC_CODERANGE_UNKNOWN unlike */
-        /* rb_enc_str_asciionly_p. It is always faster than rb_str_encode if it is available. */
-        /* Very old Rubinius (< v1.3.1) doesn't have ENC_CODERANGE_ASCIIONLY. */
-        || (rb_enc_asciicompat(rb_enc_from_index(encindex)) && ENC_CODERANGE_ASCIIONLY(v))
-#endif
-        ;
+        || (rb_enc_asciicompat(rb_enc_from_index(encindex)) && ENC_CODERANGE_ASCIIONLY(v));
 }
-#endif
 
 static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE v)
 {
@@ -425,7 +420,6 @@ static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE
         rb_raise(rb_eArgError, "size of string is too long to pack: %lu bytes should be <= %lu", len, 0xffffffffUL);
     }
 
-#ifdef COMPAT_HAVE_ENCODING
     int encindex = ENCODING_GET(v);
     if(msgpack_packer_is_binary(v, encindex) && !pk->compatibility_mode) {
         /* write ASCII-8BIT string using Binary type */
@@ -443,24 +437,11 @@ static inline void msgpack_packer_write_string_value(msgpack_packer_t* pk, VALUE
         msgpack_packer_write_raw_header(pk, (unsigned int)len);
         msgpack_buffer_append_string(PACKER_BUFFER_(pk), v);
     }
-#else
-    msgpack_packer_write_raw_header(pk, (unsigned int)len);
-    msgpack_buffer_append_string(PACKER_BUFFER_(pk), v);
-#endif
 }
 
 static inline void msgpack_packer_write_symbol_string_value(msgpack_packer_t* pk, VALUE v)
 {
-#ifdef HAVE_RB_SYM2STR
-    /* rb_sym2str is added since MRI 2.2.0 */
     msgpack_packer_write_string_value(pk, rb_sym2str(v));
-#else
-    VALUE str = rb_id2str(SYM2ID(v));
-    if (!str) {
-       rb_raise(rb_eRuntimeError, "could not convert a symbol to string");
-    }
-    msgpack_packer_write_string_value(pk, str);
-#endif
 }
 
 void msgpack_packer_write_other_value(msgpack_packer_t* pk, VALUE v);
@@ -485,9 +466,30 @@ static inline void msgpack_packer_write_fixnum_value(msgpack_packer_t* pk, VALUE
 
 static inline void msgpack_packer_write_bignum_value(msgpack_packer_t* pk, VALUE v)
 {
+    int leading_zero_bits;
+    size_t required_size = rb_absint_size(v, &leading_zero_bits);
+
     if(RBIGNUM_POSITIVE_P(v)) {
+        if(required_size > 8 && pk->has_bigint_ext_type) {
+            if(msgpack_packer_try_write_with_ext_type_lookup(pk, v)) {
+                return;
+            }
+            // if we didn't return here `msgpack_packer_write_u64` will raise a RangeError
+        }
+
         msgpack_packer_write_u64(pk, rb_big2ull(v));
     } else {
+        if(leading_zero_bits == 0) {
+            required_size += 1;
+        }
+
+        if(required_size > 8 && pk->has_bigint_ext_type) {
+            if(msgpack_packer_try_write_with_ext_type_lookup(pk, v)) {
+                return;
+            }
+            // if we didn't return here `msgpack_packer_write_u64` will raise a RangeError
+        }
+
         msgpack_packer_write_long_long(pk, rb_big2ll(v));
     }
 }