msgpack 0.4.3-mswin32 → 0.4.4-mswin32

data/ext/compat.h

@@ -0,0 +1,77 @@
+/*
+ * MessagePack for Ruby
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef COMPAT_H__
+#define COMPAT_H__
+
+
+#ifdef HAVE_RUBY_ENCODING_H
+#include "ruby/encoding.h"
+#define COMPAT_HAVE_ENCODING
+extern int s_enc_utf8;
+extern int s_enc_ascii8bit;
+extern int s_enc_usascii;
+extern VALUE s_enc_utf8_value;
+#endif
+
+#ifdef RUBY_VM
+#define COMPAT_RERAISE rb_exc_raise(rb_errinfo())
+#else
+#define COMPAT_RERAISE rb_exc_raise(ruby_errinfo)
+#endif
+
+
+/* ruby 1.8 and Rubinius */
+#ifndef RBIGNUM_POSITIVE_P
+# ifdef RUBINIUS
+#  define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
+# else
+#  define RBIGNUM_POSITIVE_P(b) (RBIGNUM(b)->sign)
+# endif
+#endif
+
+
+/* Rubinius */
+#ifdef RUBINIUS
+static inline void rb_gc_enable() { return; }
+static inline void rb_gc_disable() { return; }
+#endif
+
+
+/* ruby 1.8.5 */
+#ifndef RSTRING_PTR
+#define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#endif
+
+/* ruby 1.8.5 */
+#ifndef RSTRING_LEN
+#define RSTRING_LEN(s) (RSTRING(s)->len)
+#endif
+
+/* ruby 1.8.5 */
+#ifndef RARRAY_PTR
+#define RARRAY_PTR(s) (RARRAY(s)->ptr)
+#endif
+
+/* ruby 1.8.5 */
+#ifndef RARRAY_LEN
+#define RARRAY_LEN(s) (RARRAY(s)->len)
+#endif
+
+
+#endif /* compat.h */
+

data/ext/extconf.rb

@@ -1,5 +1,5 @@
 require 'mkmf'
 require './version.rb'
-$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\"]
+$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\" -g]
 create_makefile('msgpack')
 

data/ext/pack.c

@@ -16,6 +16,8 @@
  *    limitations under the License.
  */
 #include "ruby.h"
+#include "compat.h"
+
 #include "msgpack/pack_define.h"
 
 static ID s_to_msgpack;
@@ -116,10 +118,6 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self)
 }
 
 
-#ifndef RBIGNUM_SIGN  // Ruby 1.8
-#define RBIGNUM_SIGN(b) (RBIGNUM(b)->sign)
-#endif
-
 /*
  * Document-method: Bignum#to_msgpack
  *
@@ -131,10 +129,9 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self)
 static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
-	// FIXME bignum
-	if(RBIGNUM_SIGN(self)) {  // positive
+	if(RBIGNUM_POSITIVE_P(self)) {
 		msgpack_pack_uint64(out, rb_big2ull(self));
-	} else {  // negative
+	} else {
 		msgpack_pack_int64(out, rb_big2ll(self));
 	}
 	return out;
@@ -168,6 +165,14 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self)
 static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
+#ifdef COMPAT_HAVE_ENCODING
+	int enc = ENCODING_GET(self);
+	if(enc != s_enc_utf8 && enc != s_enc_ascii8bit && enc != s_enc_usascii) {
+		if(!ENC_CODERANGE_ASCIIONLY(self)) {
+			self = rb_str_encode(self, s_enc_utf8_value, 0, Qnil);
+		}
+	}
+#endif
 	msgpack_pack_raw(out, RSTRING_LEN(self));
 	msgpack_pack_raw_body(out, RSTRING_PTR(self), RSTRING_LEN(self));
 	return out;
@@ -184,12 +189,16 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self)
  */
 static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
+#ifdef COMPAT_HAVE_ENCODING
+	return MessagePack_String_to_msgpack(argc, argv, rb_id2str(SYM2ID(self)));
+#else
 	ARG_BUFFER(out, argc, argv);
 	const char* name = rb_id2name(SYM2ID(self));
 	size_t len = strlen(name);
 	msgpack_pack_raw(out, len);
 	msgpack_pack_raw_body(out, name, len);
 	return out;
+#endif
 }
 
 
@@ -205,7 +214,8 @@ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self)
 static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
-	msgpack_pack_array(out, RARRAY_LEN(self));
+	// FIXME check sizeof(long) > sizeof(unsigned int) && RARRAY_LEN(self) > UINT_MAX
+	msgpack_pack_array(out, (unsigned int)RARRAY_LEN(self));
 	VALUE* p = RARRAY_PTR(self);
 	VALUE* const pend = p + RARRAY_LEN(self);
 	for(;p != pend; ++p) {
@@ -239,7 +249,8 @@ static int MessagePack_Hash_to_msgpack_foreach(VALUE key, VALUE value, VALUE out
 static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
-	msgpack_pack_map(out, RHASH_SIZE(self));
+	// FIXME check sizeof(st_index_t) > sizeof(unsigned int) && RARRAY_LEN(self) > UINT_MAX
+	msgpack_pack_map(out, (unsigned int)RHASH_SIZE(self));
 	rb_hash_foreach(self, MessagePack_Hash_to_msgpack_foreach, out);
 	return out;
 }

data/ext/rbinit.c

@@ -17,9 +17,17 @@
  */
 #include "pack.h"
 #include "unpack.h"
+#include "compat.h"
 
 static VALUE mMessagePack;
 
+#ifdef COMPAT_HAVE_ENCODING
+int s_enc_utf8;
+int s_enc_ascii8bit;
+int s_enc_usascii;
+VALUE s_enc_utf8_value;
+#endif
+
 /**
  * Document-module: MessagePack
  *
@@ -46,6 +54,13 @@ void Init_msgpack(void)
 
 	rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION));
 
+#ifdef COMPAT_HAVE_ENCODING
+	s_enc_ascii8bit = rb_ascii8bit_encindex();
+	s_enc_utf8 = rb_utf8_encindex();
+	s_enc_usascii = rb_usascii_encindex();
+	s_enc_utf8_value = rb_enc_from_encoding(rb_utf8_encoding());
+#endif
+
 	Init_msgpack_unpack(mMessagePack);
 	Init_msgpack_pack(mMessagePack);
 }

data/ext/unpack.c

@@ -16,19 +16,15 @@
  *    limitations under the License.
  */
 #include "ruby.h"
+#include "compat.h"
 
 #include "msgpack/unpack_define.h"
 
 static ID s_sysread;
 static ID s_readpartial;
 
-#ifdef HAVE_RUBY_ENCODING_H
-#include "ruby/encoding.h"
-int s_ascii_8bit;
-#endif
-
 struct unpack_buffer {
-	size_t size;
+	size_t used;
 	size_t free;
 	char* ptr;
 };
@@ -41,6 +37,7 @@ typedef struct {
 	VALUE stream;
 	VALUE streambuf;
 	ID stream_append_method;
+	size_t buffer_free_size;
 } unpack_user;
 
 
@@ -136,6 +133,9 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha
 	} else {
 		*o = rb_str_substr(u->source, p - b, l);
 	}
+#ifdef COMPAT_HAVE_ENCODING
+	ENCODING_SET(*o, s_enc_utf8);
+#endif
 	return 0;
 }
 
@@ -156,27 +156,11 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha
 		rb_raise(rb_eTypeError, "instance of String needed"); \
 	}
 
-#ifdef RUBY_VM
-#define RERAISE rb_exc_raise(rb_errinfo())
-#else
-#define RERAISE rb_exc_raise(ruby_errinfo)
-#endif
-
-
-#ifdef HAVE_RUBY_ENCODING_H
-static VALUE template_execute_rescue_enc(VALUE data)
-{
-	rb_gc_enable();
-	VALUE* resc = (VALUE*)data;
-	rb_enc_set_index(resc[0], (int)resc[1]);
-	RERAISE;
-}
-#endif
 
 static VALUE template_execute_rescue(VALUE nouse)
 {
 	rb_gc_enable();
-	RERAISE;
+	COMPAT_RERAISE;
 }
 
 static VALUE template_execute_do(VALUE argv)
@@ -203,31 +187,16 @@ static int template_execute_wrap(msgpack_unpack_t* mp,
 		(VALUE)from,
 	};
 
-#ifdef HAVE_RUBY_ENCODING_H
-	int enc_orig = rb_enc_get_index(str);
-	rb_enc_set_index(str, s_ascii_8bit);
-#endif
-
 	// FIXME execute実行中はmp->topが更新されないのでGC markが機能しない
 	rb_gc_disable();
 
 	mp->user.source = str;
 
-#ifdef HAVE_RUBY_ENCODING_H
-	VALUE resc[2] = {str, enc_orig};
-	int ret = (int)rb_rescue(template_execute_do, (VALUE)args,
-			template_execute_rescue_enc, (VALUE)resc);
-#else
 	int ret = (int)rb_rescue(template_execute_do, (VALUE)args,
 			template_execute_rescue, Qnil);
-#endif
 
 	rb_gc_enable();
 
-#ifdef HAVE_RUBY_ENCODING_H
-	rb_enc_set_index(str, enc_orig);
-#endif
-
 	return ret;
 }
 
@@ -273,6 +242,13 @@ static VALUE eUnpackError;
 #define MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE (8*1024)
 #endif
 
+/*
+#ifndef MSGPACK_BUFFER_FREE_SIZE
+#define MSGPACK_BUFFER_FREE_SIZE (1024*1024)
+#endif
+*/
+#define MSGPACK_BUFFER_FREE_SIZE 0
+
 static void MessagePack_Unpacker_free(void* data)
 {
 	if(data) {
@@ -305,7 +281,7 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass)
 
 	mp->user.finished = 0;
 	mp->user.offset = 0;
-	mp->user.buffer.size = 0;
+	mp->user.buffer.used = 0;
 	mp->user.buffer.free = 0;
 	mp->user.buffer.ptr = NULL;
 
@@ -358,6 +334,7 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self)
 	mp->user.stream = stream;
 	mp->user.streambuf = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE);
 	mp->user.stream_append_method = append_method_of(stream);
+	mp->user.buffer_free_size = MSGPACK_BUFFER_FREE_SIZE;
 
 	return self;
 }
@@ -398,22 +375,27 @@ static void reserve_buffer(msgpack_unpack_t* mp, size_t require)
 {
 	struct unpack_buffer* buffer = &mp->user.buffer;
 
-	if(buffer->size == 0) {
-		size_t nsize = MSGPACK_UNPACKER_BUFFER_INIT_SIZE;
+	if(buffer->used == 0) {
+		if(require <= buffer->free) {
+			/* enough free space */
+			return;
+		}
+		/* no used buffer: realloc only */
+		size_t nsize = buffer->free == 0 ?
+			MSGPACK_UNPACKER_BUFFER_INIT_SIZE : buffer->free*2;
 		while(nsize < require) {
 			nsize *= 2;
 		}
-		char* tmp = ALLOC_N(char, nsize);
-		buffer->ptr = tmp;
+		char* tmp = REALLOC_N(buffer->ptr, char, nsize);
 		buffer->free = nsize;
-		buffer->size = 0;
+		buffer->ptr = tmp;
 		return;
 	}
 
-	if(buffer->size <= mp->user.offset) {
+	if(buffer->used <= mp->user.offset) {
 		/* clear buffer and rewind offset */
-		buffer->free += buffer->size;
-		buffer->size = 0;
+		buffer->free += buffer->used;
+		buffer->used = 0;
 		mp->user.offset = 0;
 	}
 
@@ -422,41 +404,91 @@ static void reserve_buffer(msgpack_unpack_t* mp, size_t require)
 		return;
 	}
 
-	size_t nsize = (buffer->size + buffer->free) * 2;
+	size_t nsize = (buffer->used + buffer->free) * 2;
 
-	if(mp->user.offset <= buffer->size / 2) {
+	if(mp->user.offset <= buffer->used / 2) {
 		/* parsed less than half: realloc only */
-		while(nsize < buffer->size + require) {
+		while(nsize < buffer->used + require) {
 			nsize *= 2;
 		}
 		char* tmp = REALLOC_N(buffer->ptr, char, nsize);
-		buffer->free = nsize - buffer->size;
+		buffer->free = nsize - buffer->used;
 		buffer->ptr = tmp;
 
 	} else {
 		/* parsed more than half: realloc and move */
-		size_t not_parsed = buffer->size - mp->user.offset;
+		size_t not_parsed = buffer->used - mp->user.offset;
 		while(nsize < not_parsed + require) {
 			nsize *= 2;
 		}
 		char* tmp = REALLOC_N(buffer->ptr, char, nsize);
 		memcpy(tmp, tmp + mp->user.offset, not_parsed);
-		buffer->free = nsize - buffer->size;
-		buffer->size = not_parsed;
+		buffer->free = nsize - not_parsed;
+		buffer->used = not_parsed;
 		buffer->ptr = tmp;
 		mp->user.offset = 0;
 	}
 }
 
-static inline void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len)
+static inline void try_free_buffer(msgpack_unpack_t* mp, size_t require)
 {
+	if(mp->user.buffer_free_size == 0) {
+		return;
+	}
+
 	struct unpack_buffer* buffer = &mp->user.buffer;
+	size_t csize = buffer->used + buffer->free;
 
-	if(buffer->free < len) {
-		reserve_buffer(mp, len);
+	if(csize <= mp->user.buffer_free_size) {
+		return;
 	}
-	memcpy(buffer->ptr + buffer->size, ptr, len);
-	buffer->size += len;
+
+	if(mp->user.offset <= buffer->used / 2) {
+		/* parsed less than half: do nothing */
+
+	} else if(mp->user.offset < buffer->used) {
+		/* parsed more than half but not all: realloc and move */
+		size_t nsize = MSGPACK_UNPACKER_BUFFER_INIT_SIZE;
+		size_t not_parsed = buffer->used - mp->user.offset;
+		while(nsize < not_parsed + require) {
+			nsize *= 2;
+		}
+
+		if(nsize >= csize) {
+			return;
+		}
+
+		char* tmp;
+		if(mp->user.offset == 0) {
+			tmp = ALLOC_N(char, nsize);
+			memcpy(tmp, buffer->ptr + mp->user.offset, not_parsed);
+			free(buffer->ptr);
+		} else {
+			tmp = REALLOC_N(buffer->ptr, char, nsize);
+		}
+		buffer->free = nsize - not_parsed;
+		buffer->used = not_parsed;
+		buffer->ptr = tmp;
+		mp->user.offset = 0;
+
+	} else {
+		/* all parsed: free all */
+		free(buffer->ptr);
+		buffer->free = 0;
+		buffer->used = 0;
+		buffer->ptr = NULL;
+		mp->user.offset = 0;
+	}
+}
+
+static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len)
+{
+	struct unpack_buffer* buffer = &mp->user.buffer;
+
+	reserve_buffer(mp, len);
+
+	memcpy(buffer->ptr + buffer->used, ptr, len);
+	buffer->used += len;
 	buffer->free -= len;
 }
 
@@ -530,7 +562,7 @@ static VALUE MessagePack_Unpacker_each(VALUE self)
 #endif
 
 	while(1) {
-		if(mp->user.buffer.size <= mp->user.offset) {
+		if(mp->user.buffer.used <= mp->user.offset) {
 			do_fill:
 			{
 				VALUE len = MessagePack_Unpacker_fill(self);
@@ -541,7 +573,7 @@ static VALUE MessagePack_Unpacker_each(VALUE self)
 		}
 
 		ret = template_execute_wrap_each(mp,
-				mp->user.buffer.ptr, mp->user.buffer.size,
+				mp->user.buffer.ptr, mp->user.buffer.used,
 				&mp->user.offset);
 
 		if(ret < 0) {
@@ -557,9 +589,131 @@ static VALUE MessagePack_Unpacker_each(VALUE self)
 		}
 	}
 
+	try_free_buffer(mp, 0);
+
 	return Qnil;
 }
 
+/**
+ * Document-method: MessagePack::Unpacker#feed_each
+ *
+ * call-seq:
+ *   unpacker.feed_each(data) {|object| }
+ *
+ * Same as feed(data) + each {|object| }, but tries to avoid copying of the buffer.
+ */
+static VALUE feed_each_impl(VALUE args)
+{
+	VALUE self = ((VALUE*)args)[0];
+	VALUE data = ((VALUE*)args)[1];
+	size_t* pconsumed = (size_t*)((VALUE*)args)[2];
+
+	UNPACKER(self, mp);
+	int ret;
+	const char* ptr = RSTRING_PTR(data);
+	size_t len = RSTRING_LEN(data);
+
+	if(mp->user.buffer.used > 0) {
+		while(1) {
+			ret = template_execute_wrap_each(mp,
+					mp->user.buffer.ptr, mp->user.buffer.used,
+					&mp->user.offset);
+
+			if(ret < 0) {
+				rb_raise(eUnpackError, "parse error.");
+
+			} else if(ret > 0) {
+				VALUE data = template_data(mp);
+				template_init(mp);
+				rb_yield(data);
+
+			} else {
+				break;
+			}
+		}
+	}
+
+	if(len <= 0) {
+		return Qnil;
+	}
+
+	if(mp->user.buffer.used <= mp->user.offset) {
+		// wrap & execute & feed
+		while(1) {
+			ret = template_execute_wrap_each(mp,
+					ptr, len, pconsumed);
+
+			if(ret < 0) {
+				rb_raise(eUnpackError, "parse error.");
+
+			} else if(ret > 0) {
+				VALUE data = template_data(mp);
+				template_init(mp);
+				rb_yield(data);
+
+			} else {
+				break;
+			}
+		}
+
+	} else {
+		// feed & execute
+		feed_buffer(mp, ptr, len);
+		*pconsumed = len;
+
+		while(1) {
+			ret = template_execute_wrap_each(mp,
+					mp->user.buffer.ptr, mp->user.buffer.used,
+					&mp->user.offset);
+
+			if(ret < 0) {
+				rb_raise(eUnpackError, "parse error.");
+
+			} else if(ret > 0) {
+				VALUE data = template_data(mp);
+				template_init(mp);
+				rb_yield(data);
+
+			} else {
+				break;
+			}
+		}
+	}
+
+	return Qnil;
+}
+
+static VALUE feed_each_ensure(VALUE args) {
+	VALUE self = ((VALUE*)args)[0];
+	VALUE data = ((VALUE*)args)[1];
+	size_t* pconsumed = (size_t*)((VALUE*)args)[2];
+
+	const char* dptr = RSTRING_PTR(data) + *pconsumed;
+	size_t dlen = RSTRING_LEN(data) - *pconsumed;
+
+	if(dlen > 0) {
+		UNPACKER(self, mp);
+		try_free_buffer(mp, dlen);
+		feed_buffer(mp, dptr, dlen);
+	}
+
+	return Qnil;
+}
+
+static VALUE MessagePack_Unpacker_feed_each(VALUE self, VALUE data)
+{
+	size_t consumed = 0;
+	StringValue(data);
+
+	VALUE args[3];
+	args[0] = self;
+	args[1] = data;
+	args[2] = (VALUE)&consumed;
+
+	return rb_ensure(feed_each_impl, (VALUE)args,
+			feed_each_ensure, (VALUE)args);
+}
+
 
 static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned long dlen)
 {
@@ -652,7 +806,7 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data,
  *
  * This method doesn't use the internal buffer.
  *
- * Call *reset()* method before calling this method again.
+ * Call *reset* method before calling this method again.
  *
  * UnpackError is throw when parse error is occured.
  */
@@ -674,7 +828,7 @@ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data,
  *
  * This method doesn't use the internal buffer.
  *
- * Call *reset()* method before calling this method again.
+ * Call *reset* method before calling this method again.
  *
  * This returns offset that was parsed to.
  * Use *finished?* method to check an object is deserialized and call *data*
@@ -737,6 +891,7 @@ static VALUE MessagePack_Unpacker_reset(VALUE self)
 	UNPACKER(self, mp);
 	template_init(mp);
 	mp->user.finished = 0;
+	try_free_buffer(mp, 0);
 	return self;
 }
 
@@ -746,10 +901,6 @@ void Init_msgpack_unpack(VALUE mMessagePack)
 	s_sysread = rb_intern("sysread");
 	s_readpartial = rb_intern("readpartial");
 
-#ifdef HAVE_RUBY_ENCODING_H
-	s_ascii_8bit = rb_enc_find_index("ASCII-8BIT");
-#endif
-
 	eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError);
 	cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject);
 	rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc);
@@ -762,6 +913,7 @@ void Init_msgpack_unpack(VALUE mMessagePack)
 	rb_define_method(cUnpacker, "each", MessagePack_Unpacker_each, 0);
 	rb_define_method(cUnpacker, "stream", MessagePack_Unpacker_stream_get, 0);
 	rb_define_method(cUnpacker, "stream=", MessagePack_Unpacker_stream_set, 1);
+	rb_define_method(cUnpacker, "feed_each", MessagePack_Unpacker_feed_each, 1);
 
 	/* Unbuffered API */
 	rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2);