msgpack 0.3.9 → 0.4.0

data/ext/pack.c

@@ -51,6 +51,15 @@ static ID s_append;
 		rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc); \
 	}
 
+
+/*
+ * Document-method: NilClass#to_msgpack
+ *
+ * call-seq:
+ *   nil.to_msgpack(out = '') -> String
+ *
+ * Serializes the nil into raw bytes.
+ */
 static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -58,6 +67,15 @@ static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self)
 	return out;
 }
 
+
+/*
+ * Document-method: TrueClass#to_msgpack
+ *
+ * call-seq:
+ *   true.to_msgpack(out = '') -> String
+ *
+ * Serializes the true into raw bytes.
+ */
 static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -65,6 +83,15 @@ static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self)
 	return out;
 }
 
+
+/*
+ * Document-method: FalseClass#to_msgpack
+ *
+ * call-seq:
+ *   false.to_msgpack(out = '') -> String
+ *
+ * Serializes false into raw bytes.
+ */
 static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -73,6 +100,14 @@ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self
 }
 
 
+/*
+ * Document-method: Fixnum#to_msgpack
+ *
+ * call-seq:
+ *   fixnum.to_msgpack(out = '') -> String
+ *
+ * Serializes the Fixnum into raw bytes.
+ */
 static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -85,6 +120,14 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self)
 #define RBIGNUM_SIGN(b) (RBIGNUM(b)->sign)
 #endif
 
+/*
+ * Document-method: Bignum#to_msgpack
+ *
+ * call-seq:
+ *   bignum.to_msgpack(out = '') -> String
+ *
+ * Serializes the Bignum into raw bytes.
+ */
 static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -97,6 +140,15 @@ static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self)
 	return out;
 }
 
+
+/*
+ * Document-method: Float#to_msgpack
+ *
+ * call-seq:
+ *   float.to_msgpack(out = '') -> String
+ *
+ * Serializes the Float into raw bytes.
+ */
 static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -104,6 +156,15 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self)
 	return out;
 }
 
+
+/*
+ * Document-method: String#to_msgpack
+ *
+ * call-seq:
+ *   string.to_msgpack(out = '') -> String
+ *
+ * Serializes the String into raw bytes.
+ */
 static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -112,6 +173,15 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self)
 	return out;
 }
 
+
+/*
+ * Document-method: Symbol#to_msgpack
+ *
+ * call-seq:
+ *   symbol.to_msgpack(out = '') -> String
+ *
+ * Serializes the Symbol into raw bytes.
+ */
 static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -122,6 +192,16 @@ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self)
 	return out;
 }
 
+
+/*
+ * Document-method: Array#to_msgpack
+ *
+ * call-seq:
+ *   array.to_msgpack(out = '') -> String
+ *
+ * Serializes the Array into raw bytes.
+ * This calls to_msgpack method reflectively for internal elements.
+ */
 static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -134,6 +214,7 @@ static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self)
 	return out;
 }
 
+
 #ifndef RHASH_SIZE  // Ruby 1.8
 #define RHASH_SIZE(h) (RHASH(h)->tbl ? RHASH(h)->tbl->num_entries : 0)
 #endif
@@ -146,6 +227,15 @@ static int MessagePack_Hash_to_msgpack_foreach(VALUE key, VALUE value, VALUE out
 	return ST_CONTINUE;
 }
 
+/*
+ * Document-method: Hash#to_msgpack
+ *
+ * call-seq:
+ *   hash.to_msgpack(out = '') -> String
+ *
+ * Serializes the Hash into raw bytes.
+ * This calls to_msgpack method reflectively for internal keys and values.
+ */
 static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
@@ -155,6 +245,17 @@ static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self)
 }
 
 
+/**
+ * Document-method: MessagePack.pack
+ *
+ * call-seq:
+ *   MessagePack.pack(object, out = '') -> String
+ *
+ * Serializes the object into raw bytes. The encoding of the string is ASCII-8BIT on Ruby 1.9.
+ * This method is same as object.to_msgpack(out = '').
+ *
+ * _out_ is an object that implements *<<* method like String or IO.
+ */
 static VALUE MessagePack_pack(int argc, VALUE* argv, VALUE self)
 {
 	VALUE out;
@@ -173,16 +274,22 @@ void Init_msgpack_pack(VALUE mMessagePack)
 {
 	s_to_msgpack = rb_intern("to_msgpack");
 	s_append = rb_intern("<<");
-	rb_define_method_id(rb_cNilClass,   s_to_msgpack, MessagePack_NilClass_to_msgpack, -1);
-	rb_define_method_id(rb_cTrueClass,  s_to_msgpack, MessagePack_TrueClass_to_msgpack, -1);
-	rb_define_method_id(rb_cFalseClass, s_to_msgpack, MessagePack_FalseClass_to_msgpack, -1);
-	rb_define_method_id(rb_cFixnum, s_to_msgpack, MessagePack_Fixnum_to_msgpack, -1);
-	rb_define_method_id(rb_cBignum, s_to_msgpack, MessagePack_Bignum_to_msgpack, -1);
-	rb_define_method_id(rb_cFloat,  s_to_msgpack, MessagePack_Float_to_msgpack, -1);
-	rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1);
-	rb_define_method_id(rb_cArray,  s_to_msgpack, MessagePack_Array_to_msgpack, -1);
-	rb_define_method_id(rb_cHash,   s_to_msgpack, MessagePack_Hash_to_msgpack, -1);
-	rb_define_method_id(rb_cSymbol, s_to_msgpack, MessagePack_Symbol_to_msgpack, -1);
+
+	rb_define_method(rb_cNilClass,   "to_msgpack", MessagePack_NilClass_to_msgpack, -1);
+	rb_define_method(rb_cTrueClass,  "to_msgpack", MessagePack_TrueClass_to_msgpack, -1);
+	rb_define_method(rb_cFalseClass, "to_msgpack", MessagePack_FalseClass_to_msgpack, -1);
+	rb_define_method(rb_cFixnum, "to_msgpack", MessagePack_Fixnum_to_msgpack, -1);
+	rb_define_method(rb_cBignum, "to_msgpack", MessagePack_Bignum_to_msgpack, -1);
+	rb_define_method(rb_cFloat,  "to_msgpack", MessagePack_Float_to_msgpack, -1);
+	rb_define_method(rb_cString, "to_msgpack", MessagePack_String_to_msgpack, -1);
+	rb_define_method(rb_cArray,  "to_msgpack", MessagePack_Array_to_msgpack, -1);
+	rb_define_method(rb_cHash,   "to_msgpack", MessagePack_Hash_to_msgpack, -1);
+	rb_define_method(rb_cSymbol, "to_msgpack", MessagePack_Symbol_to_msgpack, -1);
+
+	/**
+	 * MessagePack module is defined in rbinit.c file.
+	 * mMessagePack = rb_define_module("MessagePack");
+	 */
 	rb_define_module_function(mMessagePack, "pack", MessagePack_pack, -1);
 }
 

data/ext/pack.h

@@ -1,7 +1,7 @@
 /*
  * MessagePack for Ruby packing routine
  *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
  *
  *    Licensed under the Apache License, Version 2.0 (the "License");
  *    you may not use this file except in compliance with the License.

data/ext/rbinit.c

@@ -1,7 +1,7 @@
 /*
  * MessagePack for Ruby
  *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
  *
  *    Licensed under the Apache License, Version 2.0 (the "License");
  *    you may not use this file except in compliance with the License.
@@ -20,6 +20,26 @@
 
 static VALUE mMessagePack;
 
+/**
+ * Document-module: MessagePack
+ *
+ * MessagePack is a binary-based efficient object serialization library.
+ * It enables to exchange structured objects between many languages like JSON.
+ * But unlike JSON, it is very fast and small.
+ *
+ * You can install MessagePack with rubygems.
+ *
+ *   gem install msgpack
+ *
+ * Simple usage is as follows:
+ *
+ *   require 'msgpack'
+ *   msg = [1,2,3].to_msgpack  #=> "\x93\x01\x02\x03"
+ *   MessagePack.unpack(msg)   #=> [1,2,3]
+ *
+ * Use Unpacker class for streaming deserialization.
+ *
+ */
 void Init_msgpack(void)
 {
 	mMessagePack = rb_define_module("MessagePack");

data/ext/unpack.c

@@ -27,11 +27,17 @@ static ID s_readpartial;
 int s_ascii_8bit;
 #endif
 
+struct unpack_buffer {
+	size_t size;
+	size_t free;
+	char* ptr;
+};
+
 typedef struct {
 	int finished;
 	VALUE source;
 	size_t offset;
-	VALUE buffer;
+	struct unpack_buffer buffer;
 	VALUE stream;
 	VALUE streambuf;
 	ID stream_append_method;
@@ -124,7 +130,14 @@ static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k,
 #endif
 
 static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o)
-{ *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; }
+{
+	if(u->source == Qnil || l <= COW_MIN_SIZE) {
+		*o = rb_str_new(p, l);
+	} else {
+		*o = rb_str_substr(u->source, p - b, l);
+	}
+	return 0;
+}
 
 
 #include "msgpack/unpack_template.h"
@@ -203,20 +216,58 @@ static int template_execute_wrap(msgpack_unpack_t* mp,
 	return ret;
 }
 
+static int template_execute_wrap_each(msgpack_unpack_t* mp,
+		const char* ptr, size_t dlen, size_t* from)
+{
+	VALUE args[4] = {
+		(VALUE)mp,
+		(VALUE)ptr,
+		(VALUE)dlen,
+		(VALUE)from,
+	};
+
+	// FIXME execute実行中はmp->topが更新されないのでGC markが機能しない
+	rb_gc_disable();
+
+	int ret = (int)rb_rescue(template_execute_do, (VALUE)args,
+			template_execute_rescue, Qnil);
+
+	rb_gc_enable();
+
+	return ret;
+}
+
 
 static VALUE cUnpacker;
+
+
+/**
+ * Document-module: MessagePack::UnpackerError
+ *
+ */
 static VALUE eUnpackError;
 
 
+#ifndef MSGPACK_UNPACKER_BUFFER_INIT_SIZE
+#define MSGPACK_UNPACKER_BUFFER_INIT_SIZE (32*1024)
+#endif
+
+#ifndef MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE
+#define MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE (8*1024)
+#endif
+
 static void MessagePack_Unpacker_free(void* data)
 {
-	if(data) { free(data); }
+	if(data) {
+		msgpack_unpack_t* mp = (msgpack_unpack_t*)data;
+		free(mp->user.buffer.ptr);
+		free(mp);
+	}
 }
 
 static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp)
 {
 	unsigned int i;
-	rb_gc_mark(mp->user.buffer);
 	rb_gc_mark(mp->user.stream);
 	rb_gc_mark(mp->user.streambuf);
 	for(i=0; i < mp->top; ++i) {
@@ -243,6 +294,22 @@ static ID append_method_of(VALUE stream)
 	}
 }
 
+/**
+ * Document-method: MessagePack::Unpacker#initialize
+ *
+ * call-seq:
+ *   MessagePack::Unpacker.new(stream = nil)
+ *
+ * Creates instance of MessagePack::Unpacker.
+ *
+ * You can specify a _stream_ for input stream.
+ * It is required to implement *sysread* or *readpartial* method.
+ *
+ * With the input stream, buffers will be feeded into the deserializer automatically.
+ *
+ * Without the input stream, use *feed* method manually. Or you can manage the buffer manually
+ * with *execute*, *finished?*, *data* and *reset* methods.
+ */
 static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self)
 {
 	VALUE stream;
@@ -261,19 +328,38 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self)
 	template_init(mp);
 	mp->user.finished = 0;
 	mp->user.offset = 0;
-	mp->user.buffer = rb_str_new("",0);
+	mp->user.buffer.size = 0;
+	mp->user.buffer.free = 0;
+	mp->user.buffer.ptr = NULL;
 	mp->user.stream = stream;
-	mp->user.streambuf = rb_str_new("",0);
+	mp->user.streambuf = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE);
 	mp->user.stream_append_method = append_method_of(stream);
 	return self;
 }
 
+
+/**
+ * Document-method: MessagePack::Unpacker#stream
+ *
+ * call-seq:
+ *   unpacker.stream
+ *
+ * Gets the input stream.
+ */
 static VALUE MessagePack_Unpacker_stream_get(VALUE self)
 {
 	UNPACKER(self, mp);
 	return mp->user.stream;
 }
 
+/**
+ * Document-method: MessagePack::Unpacker#stream=
+ *
+ * call-seq:
+ *   unpacker.stream = stream
+ *
+ * Resets the input stream. You can set nil not to use input stream.
+ */
 static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val)
 {
 	UNPACKER(self, mp);
@@ -282,14 +368,126 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val)
 	return val;
 }
 
+
+#ifdef RUBY_VM
+#  ifndef STR_SHARED
+#    define STR_SHARED  FL_USER2
+#  endif
+#  ifndef STR_NOEMBED
+#    define STR_NOEMBED FL_USER1
+#  endif
+#  ifndef STR_ASSOC
+#    define STR_ASSOC   FL_USER3
+#  endif
+#  ifndef STR_NOCAPA_P
+#    define STR_NOCAPA_P(s) (FL_TEST(s,STR_NOEMBED) && FL_ANY(s,STR_SHARED|STR_ASSOC))
+#  endif
+#  define NEED_MORE_CAPA(s,size) (!STR_NOCAPA_P(s) && RSTRING(s)->as.heap.aux.capa < size)
+#else
+#  ifndef STR_NOCAPA
+#    ifndef STR_ASSOC
+#      define STR_ASSOC   FL_USER3
+#    endif
+#    ifndef ELTS_SHARED
+#      define ELTS_SHARED FL_USER2
+#    endif
+#    define STR_NOCAPA  (ELTS_SHARED|STR_ASSOC)
+#  endif
+#  define NEED_MORE_CAPA(s,size) (!FL_TEST(s,STR_NOCAPA) && RSTRING(s)->aux.capa < size)
+#endif
+
+static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len)
+{
+	struct unpack_buffer* buffer = &mp->user.buffer;
+
+	if(buffer->size == 0) {
+		char* tmp = (char*)malloc(MSGPACK_UNPACKER_BUFFER_INIT_SIZE);
+		// FIXME check tmp == NULL
+		buffer->ptr = tmp;
+		buffer->free = MSGPACK_UNPACKER_BUFFER_INIT_SIZE;
+		buffer->size = 0;
+
+	} else if(buffer->size <= mp->user.offset) {
+		/* clear buffer and rewind offset */
+		buffer->free += buffer->size;
+		buffer->size = 0;
+		mp->user.offset = 0;
+	}
+
+	if(len <= buffer->free) {
+		/* enough free space: just copy */
+		memcpy(buffer->ptr+buffer->size, ptr, len);
+		buffer->size += len;
+		buffer->free -= len;
+		return;
+	}
+
+	size_t csize = buffer->size + buffer->free;
+
+	if(mp->user.offset <= buffer->size / 2) {
+		/* parsed less than half: realloc and copy */
+		csize *= 2;
+		while(csize < buffer->size + len) {
+			csize *= 2;
+		}
+		char* tmp = (char*)realloc(buffer->ptr, csize);
+		// FIXME check tmp == NULL
+		memcpy(tmp + buffer->size, ptr, len);
+		buffer->ptr = tmp;
+		buffer->free = csize - buffer->size;
+		return;
+	}
+
+	size_t not_parsed = buffer->size - mp->user.offset;
+
+	if(csize < not_parsed + len) {
+		/* more buffer size */
+		csize *= 2;
+		while(csize < not_parsed + len) {
+			csize *= 2;
+		}
+		char* tmp = (char*)realloc(buffer->ptr, csize);
+		// FIXME check tmp == NULL
+		buffer->ptr = tmp;
+	}
+
+	memcpy(buffer->ptr+not_parsed, ptr, not_parsed);
+	buffer->size = not_parsed;
+	buffer->free = csize - buffer->size;
+	buffer->ptr = buffer->ptr;
+	mp->user.offset = 0;
+}
+
+/**
+ * Document-method: MessagePack::Unpacker#feed
+ *
+ * call-seq:
+ *   unpacker.feed(data)
+ *
+ * Fills the internal buffer with the specified buffer.
+ */
 static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data)
 {
 	UNPACKER(self, mp);
 	StringValue(data);
-	rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data));
+	feed_buffer(mp, RSTRING_PTR(data), RSTRING_LEN(data));
 	return Qnil;
 }
 
+/**
+ * Document-method: MessagePack::Unpacker#fill
+ *
+ * call-seq:
+ *   unpacker.fill -> length of read data
+ *
+ * Fills the internal buffer using the input stream.
+ *
+ * If the input stream is not specified, it returns nil.
+ * You can set it on *initialize* or *stream=* methods.
+ *
+ * This methods raises exceptions that _stream.sysread_ or
+ * _stream.readpartial_ method raises.
+ */
 static VALUE MessagePack_Unpacker_fill(VALUE self)
 {
 	UNPACKER(self, mp);
@@ -298,21 +496,28 @@ static VALUE MessagePack_Unpacker_fill(VALUE self)
 		return Qnil;
 	}
 
-	long len;
-	if(RSTRING_LEN(mp->user.buffer) == 0) {
-		rb_funcall(mp->user.stream, mp->user.stream_append_method, 2,
-				LONG2FIX(64*1024), mp->user.buffer);
-		len = RSTRING_LEN(mp->user.buffer);
-	} else {
-		rb_funcall(mp->user.stream, mp->user.stream_append_method, 2,
-				LONG2FIX(64*1024), mp->user.streambuf);
-		len = RSTRING_LEN(mp->user.streambuf);
-		rb_str_cat(mp->user.buffer, RSTRING_PTR(mp->user.streambuf), RSTRING_LEN(mp->user.streambuf));
-	}
+	rb_funcall(mp->user.stream, mp->user.stream_append_method, 2,
+			LONG2FIX(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE),
+			mp->user.streambuf);
+
+	size_t len = RSTRING_LEN(mp->user.streambuf);
+	feed_buffer(mp, RSTRING_PTR(mp->user.streambuf), len);
 
 	return LONG2FIX(len);
 }
 
+
+/**
+ * Document-method: MessagePack::Unpacker#each
+ *
+ * call-seq:
+ *   unpacker.each {|object| }
+ *
+ * Deserializes objects repeatedly. This calls *fill* method automatically.
+ *
+ * UnpackError is throw when parse error is occured.
+ * This method raises exceptions that *fill* method raises.
+ */
 static VALUE MessagePack_Unpacker_each(VALUE self)
 {
 	UNPACKER(self, mp);
@@ -323,7 +528,7 @@ static VALUE MessagePack_Unpacker_each(VALUE self)
 #endif
 
 	while(1) {
-		if(RSTRING_LEN(mp->user.buffer) <= mp->user.offset) {
+		if(mp->user.buffer.size <= mp->user.offset) {
 			do_fill:
 			{
 				VALUE len = MessagePack_Unpacker_fill(self);
@@ -333,8 +538,9 @@ static VALUE MessagePack_Unpacker_each(VALUE self)
 			}
 		}
 
-		ret = template_execute_wrap(mp, mp->user.buffer,
-				RSTRING_LEN(mp->user.buffer), &mp->user.offset);
+		ret = template_execute_wrap_each(mp,
+				mp->user.buffer.ptr, mp->user.buffer.size,
+				&mp->user.offset);
 
 		if(ret < 0) {
 			rb_raise(eUnpackError, "parse error.");
@@ -352,6 +558,7 @@ static VALUE MessagePack_Unpacker_each(VALUE self)
 	return Qnil;
 }
 
+
 static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned long dlen)
 {
 	msgpack_unpack_t mp;
@@ -376,12 +583,34 @@ static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned lon
 	}
 }
 
+/**
+ * Document-method: MessagePack::Unpacker.unpack_limit
+ *
+ * call-seq:
+ *   MessagePack::Unpacker.unpack_limit(data, limit) -> object
+ *
+ * Deserializes one object over the specified buffer upto _limit_ bytes.
+ *
+ * UnpackError is throw when parse error is occured, the buffer is insufficient
+ * to deserialize one object or there are extra bytes.
+ */
 static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit)
 {
 	CHECK_STRING_TYPE(data);
 	return MessagePack_unpack_impl(self, data, NUM2ULONG(limit));
 }
 
+/**
+ * Document-method: MessagePack::Unpacker.unpack
+ *
+ * call-seq:
+ *   MessagePack::Unpacker.unpack(data) -> object
+ *
+ * Deserializes one object over the specified buffer.
+ *
+ * UnpackError is throw when parse error is occured, the buffer is insufficient
+ * to deserialize one object or there are extra bytes.
+ */
 static VALUE MessagePack_unpack(VALUE self, VALUE data)
 {
 	CHECK_STRING_TYPE(data);
@@ -411,7 +640,20 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data,
 	}
 }
 
-/* compat */
+/**
+ * Document-method: MessagePack::Unpacker#execute_limit
+ *
+ * call-seq:
+ *   unpacker.unpack_limit(data, offset, limit) -> next offset
+ *
+ * Deserializes one object over the specified buffer from _offset_ bytes upto _limit_ bytes.
+ *
+ * This method doesn't use the internal buffer.
+ *
+ * Call *reset()* method before calling this method again.
+ *
+ * UnpackError is throw when parse error is occured.
+ */
 static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data,
 		VALUE off, VALUE limit)
 {
@@ -420,7 +662,24 @@ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data,
 			(size_t)NUM2ULONG(off), (size_t)NUM2ULONG(limit));
 }
 
-/* compat */
+/**
+ * Document-method: MessagePack::Unpacker#execute
+ *
+ * call-seq:
+ *   unpacker.unpack(data, offset) -> next offset
+ *
+ * Deserializes one object over the specified buffer from _offset_ bytes.
+ *
+ * This method doesn't use the internal buffer.
+ *
+ * Call *reset()* method before calling this method again.
+ *
+ * This returns offset that was parsed to.
+ * Use *finished?* method to check an object is deserialized and call *data*
+ * method if it returns true.
+ *
+ * UnpackError is throw when parse error is occured.
+ */
 static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off)
 {
 	CHECK_STRING_TYPE(data);
@@ -428,7 +687,16 @@ static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off)
 			(size_t)NUM2ULONG(off), (size_t)RSTRING_LEN(data));
 }
 
-/* compat */
+/**
+ * Document-method: MessagePack::Unpacker#finished?
+ *
+ * call-seq:
+ *   unpacker.finished?
+ *
+ * Returns true if an object is ready to get with data method.
+ *
+ * Use this method with execute method.
+ */
 static VALUE MessagePack_Unpacker_finished_p(VALUE self)
 {
 	UNPACKER(self, mp);
@@ -438,14 +706,30 @@ static VALUE MessagePack_Unpacker_finished_p(VALUE self)
 	return Qfalse;
 }
 
-/* compat */
+/**
+ * Document-method: MessagePack::Unpacker#data
+ *
+ * call-seq:
+ *   unpacker.data
+ *
+ * Gets the object deserialized by execute method.
+ *
+ * Use this method with execute method.
+ */
 static VALUE MessagePack_Unpacker_data(VALUE self)
 {
 	UNPACKER(self, mp);
 	return template_data(mp);
 }
 
-/* compat */
+/**
+ * Document-method: MessagePack::Unpacker#reset
+ *
+ * call-seq:
+ *   unpacker.reset
+ *
+ * Resets the internal state of the unpacker.
+ */
 static VALUE MessagePack_Unpacker_reset(VALUE self)
 {
 	UNPACKER(self, mp);
@@ -467,20 +751,97 @@ void Init_msgpack_unpack(VALUE mMessagePack)
 	eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError);
 	cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject);
 	rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc);
+
 	rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, -1);
+
+	/* Buffered API */
 	rb_define_method(cUnpacker, "feed", MessagePack_Unpacker_feed, 1);
 	rb_define_method(cUnpacker, "fill", MessagePack_Unpacker_fill, 0);
 	rb_define_method(cUnpacker, "each", MessagePack_Unpacker_each, 0);
 	rb_define_method(cUnpacker, "stream", MessagePack_Unpacker_stream_get, 0);
 	rb_define_method(cUnpacker, "stream=", MessagePack_Unpacker_stream_set, 1);
-	rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1);
-	rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2);
 
-	/* backward compatibility */
+	/* Unbuffered API */
 	rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2);
 	rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3);
 	rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0);
 	rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0);
 	rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0);
+
+	/**
+	 * MessagePack module is defined in rbinit.c file.
+	 * mMessagePack = rb_define_module("MessagePack");
+	 */
+	rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1);
+	rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2);
 }
 
+/**
+ * Document-module: MessagePack::Unpacker
+ *
+ * Deserializer class that includes Buffered API and Unbuffered API.
+ *
+ *
+ * Buffered API uses the internal buffer of the Unpacker.
+ * Following code uses Buffered API with an input stream:
+ *
+ *   # create an unpacker with input stream.
+ *   pac = MessagePack::Unpacker.new(STDIN)
+ *   
+ *   # deserialize object one after another.
+ *   pac.each {|obj|
+ *     # ...
+ *   }
+ *
+ *
+ * Following code doesn't use the input stream and feeds buffer
+ * manually. This is useful to use special stream or with
+ * event-driven I/O library.
+ *
+ *   # create an unpacker without input stream.
+ *   pac = MessagePack::Unpacker.new()
+ *   
+ *   # feed buffer to the internal buffer.
+ *   pac.feed(input_bytes)
+ *   
+ *   # deserialize object one after another.
+ *   pac.each {|obj|
+ *     # ...
+ *   }
+ *
+ *
+ * You can manage the buffer manually with the combination of
+ * *execute*, *finished?*, *data* and *reset* method.
+ *
+ *   # create an unpacker.
+ *   pac = MessagePack::Unpacker.new()
+ *   
+ *   # manage buffer and offset manually.
+ *   offset = 0
+ *   buffer = ''
+ *   
+ *   # read some data into the buffer.
+ *   buffer << [1,2,3].to_msgpack
+ *   buffer << [4,5,6].to_msgpack
+ *   
+ *   while true
+ *     offset = pac.execute(buffer, offset)
+ *   
+ *     if pac.finished?
+ *       obj = pac.data
+ *   
+ *       buffer.slice!(0, offset)
+ *       offset = 0
+ *       pac.reset
+ *   
+ *       # do something with the object
+ *       # ...
+ *
+ *       # repeat execution if there are more data.
+ *       next unless buffer.empty?
+ *     end
+ *
+ *     break
+ *   end
+ */
+

data/ext/unpack.h

@@ -1,7 +1,7 @@
 /*
  * MessagePack for Ruby unpacking routine
  *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
  *
  *    Licensed under the Apache License, Version 2.0 (the "License");
  *    you may not use this file except in compliance with the License.

data/msgpack/pack_template.h

@@ -16,12 +16,12 @@
  *    limitations under the License.
  */
 
-#ifdef __LITTLE_ENDIAN__
+#if defined(__LITTLE_ENDIAN__)
 #define TAKE8_8(d)  ((uint8_t*)&d)[0]
 #define TAKE8_16(d) ((uint8_t*)&d)[0]
 #define TAKE8_32(d) ((uint8_t*)&d)[0]
 #define TAKE8_64(d) ((uint8_t*)&d)[0]
-#elif __BIG_ENDIAN__
+#elif defined(__BIG_ENDIAN__)
 #define TAKE8_8(d)  ((uint8_t*)&d)[0]
 #define TAKE8_16(d) ((uint8_t*)&d)[1]
 #define TAKE8_32(d) ((uint8_t*)&d)[3]
@@ -377,14 +377,24 @@ msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d)
 
 msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d)
 {
-#if defined(SIZEOF_SHORT) || defined(SHRT_MAX)
-#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
 	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff
+#elif SIZEOF_SHORT == 4
 	msgpack_pack_real_int32(x, d);
 #else
 	msgpack_pack_real_int64(x, d);
 #endif
+
+#elif defined(SHRT_MAX)
+#if SHRT_MAX == 0x7fff
+	msgpack_pack_real_int16(x, d);
+#elif SHRT_MAX == 0x7fffffff
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
 #else
 if(sizeof(short) == 2) {
 	msgpack_pack_real_int16(x, d);
@@ -398,14 +408,24 @@ if(sizeof(short) == 2) {
 
 msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d)
 {
-#if defined(SIZEOF_INT) || defined(INT_MAX)
-#if SIZEOF_INT == 2 || INT_MAX == 0x7fff
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
 	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff
+#elif SIZEOF_INT == 4
 	msgpack_pack_real_int32(x, d);
 #else
 	msgpack_pack_real_int64(x, d);
 #endif
+
+#elif defined(INT_MAX)
+#if INT_MAX == 0x7fff
+	msgpack_pack_real_int16(x, d);
+#elif INT_MAX == 0x7fffffff
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
 #else
 if(sizeof(int) == 2) {
 	msgpack_pack_real_int16(x, d);
@@ -419,14 +439,24 @@ if(sizeof(int) == 2) {
 
 msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d)
 {
-#if defined(SIZEOF_LONG) || defined(LONG_MAX)
-#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
 	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL
+#elif SIZEOF_LONG == 4
 	msgpack_pack_real_int32(x, d);
 #else
 	msgpack_pack_real_int64(x, d);
 #endif
+
+#elif defined(LONG_MAX)
+#if LONG_MAX == 0x7fffL
+	msgpack_pack_real_int16(x, d);
+#elif LONG_MAX == 0x7fffffffL
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
 #else
 if(sizeof(long) == 2) {
 	msgpack_pack_real_int16(x, d);
@@ -440,14 +470,24 @@ if(sizeof(long) == 2) {
 
 msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d)
 {
-#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX)
-#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
+	msgpack_pack_real_int16(x, d);
+#elif SIZEOF_LONG_LONG == 4
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(LLONG_MAX)
+#if LLONG_MAX == 0x7fffL
 	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL
+#elif LLONG_MAX == 0x7fffffffL
 	msgpack_pack_real_int32(x, d);
 #else
 	msgpack_pack_real_int64(x, d);
 #endif
+
 #else
 if(sizeof(long long) == 2) {
 	msgpack_pack_real_int16(x, d);
@@ -461,14 +501,24 @@ if(sizeof(long long) == 2) {
 
 msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d)
 {
-#if defined(SIZEOF_SHORT) || defined(USHRT_MAX)
-#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
+	msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_SHORT == 4
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(USHRT_MAX)
+#if USHRT_MAX == 0xffffU
 	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU
+#elif USHRT_MAX == 0xffffffffU
 	msgpack_pack_real_uint32(x, d);
 #else
 	msgpack_pack_real_uint64(x, d);
 #endif
+
 #else
 if(sizeof(unsigned short) == 2) {
 	msgpack_pack_real_uint16(x, d);
@@ -482,14 +532,24 @@ if(sizeof(unsigned short) == 2) {
 
 msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d)
 {
-#if defined(SIZEOF_INT) || defined(UINT_MAX)
-#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
 	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU
+#elif SIZEOF_INT == 4
 	msgpack_pack_real_uint32(x, d);
 #else
 	msgpack_pack_real_uint64(x, d);
 #endif
+
+#elif defined(UINT_MAX)
+#if UINT_MAX == 0xffffU
+	msgpack_pack_real_uint16(x, d);
+#elif UINT_MAX == 0xffffffffU
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
 #else
 if(sizeof(unsigned int) == 2) {
 	msgpack_pack_real_uint16(x, d);
@@ -503,18 +563,28 @@ if(sizeof(unsigned int) == 2) {
 
 msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d)
 {
-#if defined(SIZEOF_LONG) || defined(ULONG_MAX)
-#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
 	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL
+#elif SIZEOF_LONG == 4
 	msgpack_pack_real_uint32(x, d);
 #else
 	msgpack_pack_real_uint64(x, d);
 #endif
+
+#elif defined(ULONG_MAX)
+#if ULONG_MAX == 0xffffUL
+	msgpack_pack_real_uint16(x, d);
+#elif ULONG_MAX == 0xffffffffUL
+	msgpack_pack_real_uint32(x, d);
 #else
-if(sizeof(unsigned int) == 2) {
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned long) == 2) {
 	msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned int) == 4) {
+} else if(sizeof(unsigned long) == 4) {
 	msgpack_pack_real_uint32(x, d);
 } else {
 	msgpack_pack_real_uint64(x, d);
@@ -524,14 +594,24 @@ if(sizeof(unsigned int) == 2) {
 
 msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d)
 {
-#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX)
-#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
 	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL
+#elif SIZEOF_LONG_LONG == 4
 	msgpack_pack_real_uint32(x, d);
 #else
 	msgpack_pack_real_uint64(x, d);
 #endif
+
+#elif defined(ULLONG_MAX)
+#if ULLONG_MAX == 0xffffUL
+	msgpack_pack_real_uint16(x, d);
+#elif ULLONG_MAX == 0xffffffffUL
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
 #else
 if(sizeof(unsigned long long) == 2) {
 	msgpack_pack_real_uint16(x, d);

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: msgpack
 version: !ruby/object:Gem::Version 
-  version: 0.3.9
+  version: 0.4.0
 platform: ruby
 authors: 
 - FURUHASHI Sadayuki
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-23 00:00:00 +09:00
+date: 2010-05-26 00:00:00 +09:00
 default_executable: 
 dependencies: []
 
@@ -19,10 +19,8 @@ executables: []
 
 extensions: 
 - ext/extconf.rb
-extra_rdoc_files: 
-- README
-- ChangeLog
-- AUTHORS
+extra_rdoc_files: []
+
 files: 
 - ext/extconf.rb
 - ext/pack.c
@@ -37,16 +35,13 @@ files:
 - msgpack/unpack_template.h
 - test/msgpack_test.rb
 - test/test_helper.rb
-- README
-- ChangeLog
-- AUTHORS
 has_rdoc: true
 homepage: http://msgpack.sourceforge.net/
 licenses: []
 
 post_install_message: 
-rdoc_options: []
-
+rdoc_options: 
+- ext
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 

data/AUTHORS

@@ -1 +0,0 @@
-FURUHASHI Sadayuki <frsyuki _at_ users.sourceforge.jp>

data/README

@@ -1,29 +0,0 @@
-
-= MessagePack
-
-
-== Description
-
-
-== Installation
-
-=== Archive Installation
-
-	rake install
-
-=== Gem Installation
-
-	gem install msgpack
-
-
-== Features/Problems
-
-
-== Synopsis
-
-
-== Copyright
-
-Author::    frsyuki <frsyuki@users.sourceforge.jp>
-Copyright:: Copyright (c) 2008-2009 frsyuki
-License::   Apache License, Version 2.0