msgpack 0.4.1 → 0.4.2

data/ext/unpack.c

@@ -386,90 +386,70 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val)
 }
 
 
-#ifdef RUBY_VM
-#  ifndef STR_SHARED
-#    define STR_SHARED  FL_USER2
-#  endif
-#  ifndef STR_NOEMBED
-#    define STR_NOEMBED FL_USER1
-#  endif
-#  ifndef STR_ASSOC
-#    define STR_ASSOC   FL_USER3
-#  endif
-#  ifndef STR_NOCAPA_P
-#    define STR_NOCAPA_P(s) (FL_TEST(s,STR_NOEMBED) && FL_ANY(s,STR_SHARED|STR_ASSOC))
-#  endif
-#  define NEED_MORE_CAPA(s,size) (!STR_NOCAPA_P(s) && RSTRING(s)->as.heap.aux.capa < size)
-#else
-#  ifndef STR_NOCAPA
-#    ifndef STR_ASSOC
-#      define STR_ASSOC   FL_USER3
-#    endif
-#    ifndef ELTS_SHARED
-#      define ELTS_SHARED FL_USER2
-#    endif
-#    define STR_NOCAPA  (ELTS_SHARED|STR_ASSOC)
-#  endif
-#  define NEED_MORE_CAPA(s,size) (!FL_TEST(s,STR_NOCAPA) && RSTRING(s)->aux.capa < size)
-#endif
-
-static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len)
+static void reserve_buffer(msgpack_unpack_t* mp, size_t require)
 {
 	struct unpack_buffer* buffer = &mp->user.buffer;
 
 	if(buffer->size == 0) {
-		char* tmp = ALLOC_N(char, MSGPACK_UNPACKER_BUFFER_INIT_SIZE);
+		size_t nsize = MSGPACK_UNPACKER_BUFFER_INIT_SIZE;
+		while(nsize < require) {
+			nsize *= 2;
+		}
+		char* tmp = ALLOC_N(char, nsize);
 		buffer->ptr = tmp;
-		buffer->free = MSGPACK_UNPACKER_BUFFER_INIT_SIZE;
+		buffer->free = nsize;
 		buffer->size = 0;
+		return;
+	}
 
-	} else if(buffer->size <= mp->user.offset) {
+	if(buffer->size <= mp->user.offset) {
 		/* clear buffer and rewind offset */
 		buffer->free += buffer->size;
 		buffer->size = 0;
 		mp->user.offset = 0;
 	}
 
-	if(len <= buffer->free) {
-		/* enough free space: just copy */
-		memcpy(buffer->ptr+buffer->size, ptr, len);
-		buffer->size += len;
-		buffer->free -= len;
+	if(require <= buffer->free) {
+		/* enough free space */
 		return;
 	}
 
-	size_t csize = buffer->size + buffer->free;
+	size_t nsize = (buffer->size + buffer->free) * 2;
 
 	if(mp->user.offset <= buffer->size / 2) {
-		/* parsed less than half: realloc and copy */
-		csize *= 2;
-		while(csize < buffer->size + len) {
-			csize *= 2;
+		/* parsed less than half: realloc only */
+		while(nsize < buffer->size + require) {
+			nsize *= 2;
 		}
-		char* tmp = REALLOC_N(buffer->ptr, char, csize);
-		memcpy(tmp + buffer->size, ptr, len);
+		char* tmp = REALLOC_N(buffer->ptr, char, nsize);
+		buffer->free = nsize - buffer->size;
 		buffer->ptr = tmp;
-		buffer->free = csize - buffer->size;
-		return;
-	}
 
-	size_t not_parsed = buffer->size - mp->user.offset;
-
-	if(csize < not_parsed + len) {
-		/* more buffer size */
-		csize *= 2;
-		while(csize < not_parsed + len) {
-			csize *= 2;
+	} else {
+		/* parsed more than half: realloc and move */
+		size_t not_parsed = buffer->size - mp->user.offset;
+		while(nsize < not_parsed + require) {
+			nsize *= 2;
 		}
-		char* tmp = REALLOC_N(buffer->ptr, char, csize);
+		char* tmp = REALLOC_N(buffer->ptr, char, nsize);
+		memcpy(tmp, tmp + mp->user.offset, not_parsed);
+		buffer->free = nsize - buffer->size;
+		buffer->size = not_parsed;
 		buffer->ptr = tmp;
+		mp->user.offset = 0;
 	}
+}
 
-	memcpy(buffer->ptr+not_parsed, ptr, not_parsed);
-	buffer->size = not_parsed;
-	buffer->free = csize - buffer->size;
-	buffer->ptr = buffer->ptr;
-	mp->user.offset = 0;
+static inline void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len)
+{
+	struct unpack_buffer* buffer = &mp->user.buffer;
+
+	if(buffer->free < len) {
+		reserve_buffer(mp, len);
+	}
+	memcpy(buffer->ptr + buffer->size, ptr, len);
+	buffer->size += len;
+	buffer->free -= len;
 }
 
 /**

data/test/cases.json

@@ -0,0 +1 @@
+[false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,127,127,255,65535,4294967295,-32,-32,-128,-32768,-2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","",[0],[0],[0],[],[],[],{},{},{},{"a":97},{"a":97},{"a":97},[[]],[["a"]]]

data/test/test_cases.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+here = File.dirname(__FILE__)
+require "#{here}/test_helper"
+
+begin
+require 'json'
+rescue LoadError
+require 'rubygems'
+require 'json'
+end
+
+CASES_PATH         = "#{here}/cases.mpac"
+CASES_COMPACT_PATH = "#{here}/cases_compact.mpac"
+CASES_JSON_PATH    = "#{here}/cases.json"
+
+class MessagePackTestCases < Test::Unit::TestCase
+	def feed_file(path)
+		pac = MessagePack::Unpacker.new
+		pac.feed File.read(path)
+		pac
+	end
+
+	def test_compare_compact
+		pac  = feed_file(CASES_PATH)
+		cpac = feed_file(CASES_COMPACT_PATH)
+
+		 objs = [];  pac.each {| obj|  objs <<  obj }
+		cobjs = []; cpac.each {|cobj| cobjs << cobj }
+
+		objs.zip(cobjs).each {|obj, cobj|
+			assert_equal(obj, cobj)
+		}
+	end
+
+	def test_compare_json
+		pac  = feed_file(CASES_PATH)
+
+		objs = []; pac.each {|obj| objs <<  obj }
+		jobjs = JSON.load File.read(CASES_JSON_PATH)
+
+		objs.zip(jobjs) {|obj, jobj|
+			assert_equal(obj, jobj)
+		}
+	end
+end
+

data/test/test_helper.rb

@@ -1,3 +1,7 @@
 require 'test/unit'
+begin
+require File.dirname(__FILE__) + '/../msgpack'
+rescue LoadError
 require File.dirname(__FILE__) + '/../lib/msgpack'
+end
 

data/test/{msgpack_test.rb → test_pack_unpack.rb}

@@ -1,8 +1,7 @@
 #!/usr/bin/env ruby
-require File.dirname(__FILE__) + '/test_helper.rb'
-require 'test/unit'
+require File.dirname(__FILE__)+'/test_helper'
 
-class MessagePackTestFormat < Test::Unit::TestCase
+class MessagePackTestPackUnpack < Test::Unit::TestCase
 	def self.it(name, &block)
 		define_method("test_#{name}", &block)
 	end
@@ -177,16 +176,18 @@ class MessagePackTestFormat < Test::Unit::TestCase
 		match ({}), "\x80"
 	end
 
-	it "{0=>0, 1=>1, ..., 14=>14}" do
-		a = (0..14).to_a;
-		match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"
-	end
-
-	it "{0=>0, 1=>1, ..., 15=>15}" do
-		a = (0..15).to_a;
-		match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"
-	end
+## FIXME
+#	it "{0=>0, 1=>1, ..., 14=>14}" do
+#		a = (0..14).to_a;
+#		match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"
+#	end
+#
+#	it "{0=>0, 1=>1, ..., 15=>15}" do
+#		a = (0..15).to_a;
+#		match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"
+#	end
 
+## FIXME
 #	it "fixmap" do
 #		check_map 1, 0
 #		check_map 1, (1<<4)-1
@@ -202,6 +203,37 @@ class MessagePackTestFormat < Test::Unit::TestCase
 #		#check_map 5, (1<<32)-1  # memory error
 #	end
 
+	it "buffer" do
+		str = "a"*32*1024*4
+		raw = str.to_msgpack
+		pac = MessagePack::Unpacker.new
+
+		len = 0
+		parsed = false
+
+		n = 655
+		time = raw.size / n
+		time += 1 unless raw.size % n == 0
+		off = 0
+
+		time.times do
+			assert(!parsed)
+
+			fe = raw[off, n]
+			assert(fe.length > 0)
+			off += fe.length
+
+			pac.feed fe
+			pac.each {|obj|
+				assert(!parsed)
+				assert_equal(obj, str)
+				parsed = true
+			}
+		end
+
+		assert(parsed)
+	end
+
 	it "gc mark" do
 		obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"]
 		num = 4

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: msgpack
 version: !ruby/object:Gem::Version 
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors: 
 - FURUHASHI Sadayuki
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-27 00:00:00 +09:00
+date: 2010-06-03 00:00:00 +09:00
 default_executable: 
 dependencies: []
 
@@ -33,8 +33,12 @@ files:
 - msgpack/sysdep.h
 - msgpack/unpack_define.h
 - msgpack/unpack_template.h
-- test/msgpack_test.rb
+- test/cases.json
+- test/cases.mpac
+- test/cases_compact.mpac
+- test/test_cases.rb
 - test/test_helper.rb
+- test/test_pack_unpack.rb
 has_rdoc: true
 homepage: http://msgpack.sourceforge.net/
 licenses: []
@@ -64,4 +68,6 @@ signing_key:
 specification_version: 3
 summary: MessagePack, a binary-based efficient data interchange format.
 test_files: 
+- test/test_cases.rb
 - test/test_helper.rb
+- test/test_pack_unpack.rb