mochilo 1.3.0 → 2.1

data/ext/mochilo/mochilo_pack.c

@@ -7,7 +7,27 @@
 
 extern VALUE rb_eMochiloPackError;
 
-void mochilo_pack_one(mochilo_buf *buf, VALUE rb_object, int trusted);
+void mochilo_pack_one(mochilo_buf *buf, VALUE rb_object);
+
+static void mochilo_buf_put_ext_size(mochilo_buf *buf, long size)
+{
+	if (size < 0x100) {
+		uint8_t lead = size;
+		mochilo_buf_putc(buf, MSGPACK_T_ENC8);
+		mochilo_buf_putc(buf, lead);
+	} else if (size < 0x10000) {
+		uint16_t lead = size;
+		mochilo_buf_putc(buf, MSGPACK_T_ENC16);
+		mochilo_buf_put16be(buf, &lead);
+	} else if (size < 0x100000000) {
+		mochilo_buf_putc(buf, MSGPACK_T_ENC32);
+		mochilo_buf_put32be(buf, &size);
+	} else {
+		// there is no ext 64
+		rb_raise(rb_eMochiloPackError,
+			"String cannot be larger than %ld bytes", 0x100000000);
+	}
+}
 
 void mochilo_pack_fixnum(mochilo_buf *buf, VALUE rb_fixnum)
 {
@@ -73,42 +93,49 @@ void mochilo_pack_bignum(mochilo_buf *buf, VALUE rb_bignum)
 	}
 }
 
-#ifdef HAVE_RUBY_ENCODING_H
+static void mochilo_put_ext_size_and_type(mochilo_buf *buf, size_t size, enum mochilo_ext_types_t type)
+{
+	mochilo_buf_put_ext_size(buf, size + 1);
+	mochilo_buf_putc(buf, MOCHILO_EXT_TYPE);
+	mochilo_buf_putc(buf, type);
+}
+
+void mochilo_pack_symbol(mochilo_buf *buf, VALUE rb_symbol)
+{
+	char *symbol_name;
+	size_t size;
+
+	symbol_name = rb_id2name(SYM2ID(rb_symbol));
+	size = strlen(symbol_name);
+
+	mochilo_put_ext_size_and_type(buf, size, MOCHILO_T_SYMBOL);
+	mochilo_buf_put(buf, symbol_name, size);
+}
+
 void mochilo_pack_regexp(mochilo_buf *buf, VALUE rb_regexp)
 {
+	uint32_t options;
 	size_t size;
 	rb_encoding *encoding;
 	char *enc_name;
 	const struct mochilo_enc_map *enc2id;
-	uint32_t options;
-	const char *regexp;
-
-	size = RREGEXP_SRC_LEN(rb_regexp);
-
-	if (size < (1<<16)) {
-		uint16_t packed_size = size;
 
-		encoding = rb_enc_get(rb_regexp);
-		enc_name = rb_enc_name(encoding);
-		enc2id = mochilo_encoding_to_id(enc_name, (unsigned int)strlen(enc_name));
+	encoding = rb_enc_get(rb_regexp);
+	enc_name = rb_enc_name(encoding);
+	enc2id = mochilo_encoding_to_id(enc_name, (unsigned int)strlen(enc_name));
 
-		options = rb_reg_options(rb_regexp);
-		regexp = RREGEXP_SRC_PTR(rb_regexp);
+	options = rb_reg_options(rb_regexp);
+	size = RREGEXP_SRC_LEN(rb_regexp);
 
-		mochilo_buf_putc(buf, MSGPACK_T_REGEXP);
-		mochilo_buf_put16be(buf, &packed_size);
-		mochilo_buf_put32be(buf, &options);
-		mochilo_buf_putc(buf, enc2id ? enc2id->id : 0);
-		mochilo_buf_put(buf, regexp, size);
-	} else {
-		rb_raise(rb_eMochiloPackError,
-			"Regexp too long: must be under %d bytes, %ld given", 1<<16, size);
-	}
+	mochilo_put_ext_size_and_type(buf, 4 + 1 + size, MOCHILO_T_REGEXP);
+	mochilo_buf_put32be(buf, &options);
+	mochilo_buf_putc(buf, enc2id ? enc2id->id : 0);
+	mochilo_buf_put(buf, RREGEXP_SRC_PTR(rb_regexp), size);
 }
-#endif
 
 void mochilo_pack_time(mochilo_buf *buf, VALUE rb_time)
 {
+	struct time_object *tobj;
 	uint64_t sec;
 	uint64_t usec;
 	int32_t utc_offset;
@@ -117,7 +144,7 @@ void mochilo_pack_time(mochilo_buf *buf, VALUE rb_time)
 	usec = NUM2ULONG(rb_funcall(rb_time, rb_intern("usec"), 0));
 	utc_offset = NUM2INT(rb_funcall(rb_time, rb_intern("utc_offset"), 0));
 
-	mochilo_buf_putc(buf, MSGPACK_T_TIME);
+	mochilo_put_ext_size_and_type(buf, 8+8+4, MOCHILO_T_TIME);
 	mochilo_buf_put64be(buf, &sec);
 	mochilo_buf_put64be(buf, &usec);
 	mochilo_buf_put32be(buf, &utc_offset);
@@ -125,14 +152,13 @@ void mochilo_pack_time(mochilo_buf *buf, VALUE rb_time)
 
 struct mochilo_hash_pack {
 	mochilo_buf *buf;
-	int trusted;
 };
 
 static int hash_callback(VALUE key, VALUE val, VALUE opaque)
 {
 	struct mochilo_hash_pack *hash_pack = (struct mochilo_hash_pack*)opaque;
-	mochilo_pack_one(hash_pack->buf, key, hash_pack->trusted);
-	mochilo_pack_one(hash_pack->buf, val, hash_pack->trusted);
+	mochilo_pack_one(hash_pack->buf, key);
+	mochilo_pack_one(hash_pack->buf, val);
 	return 0;
 }
 
@@ -143,13 +169,12 @@ void mochilo_pack_double(mochilo_buf *buf, VALUE rb_double)
 	mochilo_buf_put64be(buf, &d);
 }
 
-void mochilo_pack_hash(mochilo_buf *buf, VALUE rb_hash, int trusted)
+void mochilo_pack_hash(mochilo_buf *buf, VALUE rb_hash)
 {
 	struct mochilo_hash_pack hash_pack;
 	long size = RHASH_SIZE(rb_hash);
 
 	hash_pack.buf = buf;
-	hash_pack.trusted = trusted;
 
 	if (size < 0x10) {
 		uint8_t lead = 0x80 | size;
@@ -174,46 +199,26 @@ void mochilo_pack_bytes(mochilo_buf *buf, VALUE rb_bytes)
 {
 	long size = RSTRING_LEN(rb_bytes);
 
-	if (size < 0x20) {
-		uint8_t lead = 0xA0 | size;
+	if (size < 0x100) {
+		uint8_t lead = size;
+		mochilo_buf_putc(buf, MSGPACK_T_BIN8);
 		mochilo_buf_putc(buf, lead);
-	}
-
-	else if (size < 0x10000) {
+	} else if (size < 0x10000) {
 		uint16_t lead = size;
-		mochilo_buf_putc(buf, MSGPACK_T_RAW16);
+		mochilo_buf_putc(buf, MSGPACK_T_BIN16);
 		mochilo_buf_put16be(buf, &lead);
-	}
-
-	else {
-		mochilo_buf_putc(buf, MSGPACK_T_RAW32);
+	} else if (size < 0x100000000) {
+		mochilo_buf_putc(buf, MSGPACK_T_BIN32);
 		mochilo_buf_put32be(buf, &size);
-	}
-
-	mochilo_buf_put(buf, RSTRING_PTR(rb_bytes), size);
-}
-
-void mochilo_pack_symbol(mochilo_buf *buf, VALUE rb_symbol)
-{
-	long size;
-	const char *name;
-
-	name = rb_id2name(SYM2ID(rb_symbol));
-	size = strlen(name);
-
-	if (size < 0x100) {
-		uint8_t lead = size;
-		mochilo_buf_putc(buf, MSGPACK_T_SYM);
-		mochilo_buf_putc(buf, lead);
 	} else {
+		// there is no bin 64
 		rb_raise(rb_eMochiloPackError,
-			"Symbol too long: must be under %d bytes, %ld given", 0x100, size);
+			"Binary string cannot be larger than %ld bytes", 0x100000000);
 	}
 
-	mochilo_buf_put(buf, name, size);
+	mochilo_buf_put(buf, RSTRING_PTR(rb_bytes), size);
 }
 
-#ifdef HAVE_RUBY_ENCODING_H
 void mochilo_pack_str(mochilo_buf *buf, VALUE rb_str)
 {
 	long size = RSTRING_LEN(rb_str);
@@ -222,27 +227,41 @@ void mochilo_pack_str(mochilo_buf *buf, VALUE rb_str)
 	const struct mochilo_enc_map *enc2id;
 	const char *enc_name;
 
-	if (size < 0x10000) {
-		uint16_t lead = size;
-		mochilo_buf_putc(buf, MSGPACK_T_STR16);
-		mochilo_buf_put16be(buf, &lead);
-	}
-
-	else {
-		mochilo_buf_putc(buf, MSGPACK_T_STR32);
-		mochilo_buf_put32be(buf, &size);
-	}
-
 	encoding = rb_enc_get(rb_str);
 	enc_name = rb_enc_name(encoding);
-	enc2id = mochilo_encoding_to_id(enc_name, (unsigned int)strlen(enc_name));
 
-	mochilo_buf_putc(buf, enc2id ? enc2id->id : 0);
+	if (encoding == rb_utf8_encoding()) {
+		// use str type from msgpack spec
+		if (size < 0x20) {
+			uint8_t lead = 0xa0 | size;
+			mochilo_buf_putc(buf, lead);
+		} else if (size < 0x100) {
+			uint8_t lead = size;
+			mochilo_buf_putc(buf, MSGPACK_T_STR8);
+			mochilo_buf_putc(buf, lead);
+		} else if (size < 0x10000) {
+			uint16_t lead = size;
+			mochilo_buf_putc(buf, MSGPACK_T_STR16);
+			mochilo_buf_put16be(buf, &lead);
+		} else if (size < 0x100000000) {
+			mochilo_buf_putc(buf, MSGPACK_T_STR32);
+			mochilo_buf_put32be(buf, &size);
+		} else {
+			// there is no str 64
+			rb_raise(rb_eMochiloPackError,
+				"String cannot be larger than %ld bytes", 0x100000000);
+		}
+	} else {
+		// if another encoding is used we need to use our custom types
+		mochilo_buf_put_ext_size(buf, size);
+		enc2id = mochilo_encoding_to_id(enc_name, (unsigned int)strlen(enc_name));
+		mochilo_buf_putc(buf, enc2id ? enc2id->id : 0);
+	}
+
 	mochilo_buf_put(buf, RSTRING_PTR(rb_str), size);
 }
-#endif
 
-void mochilo_pack_array(mochilo_buf *buf, VALUE rb_array, int trusted)
+void mochilo_pack_array(mochilo_buf *buf, VALUE rb_array)
 {
 	long i, size = RARRAY_LEN(rb_array);
 
@@ -263,11 +282,11 @@ void mochilo_pack_array(mochilo_buf *buf, VALUE rb_array, int trusted)
 	}
 
 	for (i = 0; i < size; i++) {
-		mochilo_pack_one(buf, rb_ary_entry(rb_array, i), trusted);
+		mochilo_pack_one(buf, rb_ary_entry(rb_array, i));
 	}
 }
 
-void mochilo_pack_one(mochilo_buf *buf, VALUE rb_object, int trusted)
+void mochilo_pack_one(mochilo_buf *buf, VALUE rb_object)
 {
 	switch (rb_type(rb_object)) {
 	case T_NIL:
@@ -286,28 +305,19 @@ void mochilo_pack_one(mochilo_buf *buf, VALUE rb_object, int trusted)
 		mochilo_pack_fixnum(buf, rb_object);
 		return;
 
-	case T_SYMBOL:
-		if (trusted)
-			mochilo_pack_symbol(buf, rb_object);
-		else
-			mochilo_pack_str(buf, rb_obj_as_string(rb_object));
-		return;
-
 	case T_STRING:
-#ifdef HAVE_RUBY_ENCODING_H
 		if (ENCODING_GET(rb_object) != 0)
 			mochilo_pack_str(buf, rb_object);
 		else
-#endif
 			mochilo_pack_bytes(buf, rb_object);
 		return;
 
 	case T_HASH:
-		mochilo_pack_hash(buf, rb_object, trusted);
+		mochilo_pack_hash(buf, rb_object);
 		return;
 
 	case T_ARRAY:
-		mochilo_pack_array(buf, rb_object, trusted);
+		mochilo_pack_array(buf, rb_object);
 		return;
 
 	case T_FLOAT:
@@ -318,22 +328,20 @@ void mochilo_pack_one(mochilo_buf *buf, VALUE rb_object, int trusted)
 		mochilo_pack_bignum(buf, rb_object);
 		return;
 
-#ifdef HAVE_RUBY_ENCODING_H
+	case T_SYMBOL:
+		mochilo_pack_symbol(buf, rb_object);
+		return;
+
 	case T_REGEXP:
 		mochilo_pack_regexp(buf, rb_object);
 		return;
-#endif
 
 	default:
 		if (rb_cTime == rb_obj_class(rb_object)) {
 			mochilo_pack_time(buf, rb_object);
-		} else if (rb_respond_to(rb_object, rb_intern("to_bpack"))) {
-			VALUE bpack = rb_funcall(rb_object, rb_intern("to_bpack"), 0);
-
-			mochilo_buf_put(buf, RSTRING_PTR(bpack), RSTRING_LEN(bpack));
 		} else {
 			rb_raise(rb_eMochiloPackError,
-					"Unsupported object type: %s", rb_obj_classname(rb_object));
+				"Unsupported object type: %s", rb_obj_classname(rb_object));
 		}
 		return;
 	}

data/ext/mochilo/mochilo_unpack.c

@@ -6,6 +6,8 @@
 #include "mochilo.h"
 #include "mochilo_api.h"
 
+extern VALUE rb_eMochiloUnpackError;
+
 static inline int unpack_array(mo_value *_array, size_t elements, mochilo_src *buf)
 {
 	size_t i;
@@ -57,6 +59,69 @@ static inline int unpack_hash(mo_value *_hash, size_t elements, mochilo_src *buf
 	return 0; \
 }
 
+static int mochilo_unpack_custom(mo_value *_value, mochilo_src *src, size_t length)
+{
+	uint8_t custom_type;
+
+	SRC_ENSURE_AVAIL(src, length);
+
+	if (length < 1)
+		return -1;
+	length--;
+	mochilo_src_get8be(src, &custom_type);
+
+	switch (custom_type) {
+		case MOCHILO_T_SYMBOL:
+		{
+			const char *ptr;
+			if (!(ptr = mochilo_src_peek(src, length)))
+				return -1;
+			*_value = moapi_symbol_new(ptr, length);
+			return 0;
+		}
+
+		case MOCHILO_T_REGEXP:
+		{
+			uint32_t options;
+			uint8_t encoding;
+			const char *ptr;
+
+			if (length < 5)
+				return -1;
+			mochilo_src_get32be(src, &options);
+			mochilo_src_get8be(src, &encoding);
+			length -= 5;
+
+			if (!(ptr = mochilo_src_peek(src, length)))
+				return -1;
+
+			*_value = moapi_regexp_new(ptr, length, encoding, options);
+			return 0;
+		}
+
+		case MOCHILO_T_TIME:
+		{
+			uint64_t sec;
+			uint64_t usec;
+			int32_t utc_offset;
+
+			if (length != 8+8+4)
+				return -1;
+
+			mochilo_src_get64be(src, &sec);
+			mochilo_src_get64be(src, &usec);
+			mochilo_src_get32be(src, &utc_offset);
+
+			*_value = moapi_time_new(sec, usec, utc_offset);
+			return 0;
+		}
+
+		default:
+			rb_raise(rb_eMochiloUnpackError, "unknown custom type 0x%02x", custom_type);
+			return -1;
+	}
+}
+
 int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 {
 	uint8_t leader;
@@ -159,14 +224,10 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 			return unpack_hash(_value, (size_t)length, src);
 		}
 
-		case MSGPACK_T_SYM:
+		case MSGPACK_T_STR8:
 		{
 			uint8_t length;
 			const char *ptr;
-			
-			if (!src->trusted) {
-				return MSGPACK_EUNSAFE;
-			}
 
 			SRC_ENSURE_AVAIL(src, 1);
 			mochilo_src_get8be(src, &length);
@@ -174,48 +235,61 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 			if (!(ptr = mochilo_src_peek(src, length)))
 				return -1;
 
-			*_value = moapi_sym_new(ptr, length);
+			*_value = moapi_str_new(ptr, length, MSGPACK_ENC_UTF_8);
 			return 0;
 		}
 
-#ifdef HAVE_RUBY_ENCODING_H
-		case MSGPACK_T_REGEXP:
+		case MSGPACK_T_STR16:
 		{
 			uint16_t length;
-			uint32_t options;
-			uint8_t encoding;
 			const char *ptr;
 
-			SRC_ENSURE_AVAIL(src, 2+4+1);
+			SRC_ENSURE_AVAIL(src, 2 + 1);
 			mochilo_src_get16be(src, &length);
-			mochilo_src_get32be(src, &options);
-			mochilo_src_get8be(src, &encoding);
 
 			if (!(ptr = mochilo_src_peek(src, length)))
 				return -1;
 
-			*_value = moapi_regexp_new(ptr, length, encoding, options);
+			*_value = moapi_str_new(ptr, length, MSGPACK_ENC_UTF_8);
 			return 0;
 		}
-#endif
 
-		case MSGPACK_T_TIME:
+		case MSGPACK_T_STR32:
 		{
-			uint64_t sec;
-			uint64_t usec;
-			int32_t utc_offset;
+			uint32_t length;
+			const char *ptr;
 
-			SRC_ENSURE_AVAIL(src, 8+8+4);
-			mochilo_src_get64be(src, &sec);
-			mochilo_src_get64be(src, &usec);
-			mochilo_src_get32be(src, &utc_offset);
+			SRC_ENSURE_AVAIL(src, 4 + 1);
+			mochilo_src_get32be(src, &length);
 
-			*_value = moapi_time_new(sec, usec, utc_offset);
+			if (!(ptr = mochilo_src_peek(src, length)))
+				return -1;
+
+			*_value = moapi_str_new(ptr, length, MSGPACK_ENC_UTF_8);
 			return 0;
 		}
 
-#ifdef HAVE_RUBY_ENCODING_H
-		case MSGPACK_T_STR16:
+		case MSGPACK_T_ENC8:
+		{
+			uint8_t length;
+			uint8_t encoding;
+			const char *ptr;
+
+			SRC_ENSURE_AVAIL(src, 1 + 1);
+			mochilo_src_get8be(src, &length);
+			mochilo_src_get8be(src, &encoding);
+
+			if (encoding == MOCHILO_EXT_TYPE)
+				return mochilo_unpack_custom(_value, src, length);
+
+			if (!(ptr = mochilo_src_peek(src, length)))
+				return -1;
+
+			*_value = moapi_str_new(ptr, length, encoding);
+			return 0;
+		}
+
+		case MSGPACK_T_ENC16:
 		{
 			uint16_t length;
 			uint8_t encoding;
@@ -225,6 +299,9 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 			mochilo_src_get16be(src, &length);
 			mochilo_src_get8be(src, &encoding);
 
+			if (encoding == MOCHILO_EXT_TYPE)
+				return mochilo_unpack_custom(_value, src, length);
+
 			if (!(ptr = mochilo_src_peek(src, length)))
 				return -1;
 
@@ -232,7 +309,7 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 			return 0;
 		}
 
-		case MSGPACK_T_STR32:
+		case MSGPACK_T_ENC32:
 		{
 			uint32_t length;
 			uint8_t encoding;
@@ -242,15 +319,32 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 			mochilo_src_get32be(src, &length);
 			mochilo_src_get8be(src, &encoding);
 
+			if (encoding == MOCHILO_EXT_TYPE)
+				return mochilo_unpack_custom(_value, src, length);
+
 			if (!(ptr = mochilo_src_peek(src, length)))
 				return -1;
 
 			*_value = moapi_str_new(ptr, length, encoding);
 			return 0;
 		}
-#endif
 
-		case MSGPACK_T_RAW16:
+		case MSGPACK_T_BIN8:
+		{
+			uint8_t length;
+			const char *ptr;
+
+			SRC_ENSURE_AVAIL(src, 1);
+			mochilo_src_get8be(src, &length);
+
+			if (!(ptr = mochilo_src_peek(src, length)))
+				return -1;
+
+			*_value = moapi_bytes_new(ptr, length);
+			return 0;
+		}
+
+		case MSGPACK_T_BIN16:
 		{
 			uint16_t length;
 			const char *ptr;
@@ -265,7 +359,7 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 			return 0;
 		}
 
-		case MSGPACK_T_RAW32:
+		case MSGPACK_T_BIN32:
 		{
 			uint32_t length;
 			const char *ptr;
@@ -304,7 +398,7 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 				if (!(ptr = mochilo_src_peek(src, length)))
 					return -1;
 
-				*_value = moapi_bytes_new(ptr, length);
+				*_value = moapi_str_new(ptr, length, MSGPACK_ENC_UTF_8);
 				return 0;
 			}
 
@@ -312,4 +406,3 @@ int mochilo_unpack_one(mo_value *_value, mochilo_src *src)
 		}
 	}
 }
-

data/lib/mochilo/version.rb

@@ -1,3 +1,3 @@
 module Mochilo
-  VERSION = "1.3.0"
+  VERSION = "2.1"
 end

data/mochilo.gemspec

@@ -13,8 +13,9 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.4.2}
   s.summary = %q{A ruby library for BananaPack}
-  s.test_files = `git ls-files test`.split("\n")
+  s.test_files = `git ls-files spec`.split("\n")
   s.required_ruby_version = ">= 1.9.3"
+  s.licenses = ["MIT"]
 
   # tests
   s.add_development_dependency 'rake-compiler', ">= 0.8.1"

data/script/bootstrap

@@ -12,5 +12,6 @@ cd "$(dirname "$0")/.."
 if bundle check 1>/dev/null 2>&1; then
     echo "Gem environment up-to-date"
 else
+    gem update bundler
     exec bundle install --binstubs --path vendor/gems --without benchmark "$@"
 fi