bert 1.1.2 → 1.1.6

data/History.txt

@@ -1,3 +1,13 @@
+= 1.1.6 / 2012-05-25
+  * Bug fixes
+    * Better handling of utf-8 characters
+
+= 1.1.5 / 2011-12-09
+  * Bug fixes
+    * Faster and more secure C BERT decoder
+    * Fix for encoding of negative bignums
+    * Ruby 1.9 compatibility
+
 = 1.1.2 / 2010-02-08
   * Bug fixes
     * Fix bignum handling on 256 byte boundary
@@ -28,4 +38,4 @@
     * Add roundtrip tests
 
 = 0.1.0 / 2009-10-08
-  * Birthday!
+  * Birthday!

data/README.md

@@ -49,9 +49,9 @@ Usage
     require 'bert'
     
     bert = BERT.encode(t[:user, {:name => 'TPW', :nick => 'mojombo'}])
-    # => "\203h\002d\000\004userh\002d\000\004dictl\000\000\000\002h\002d
-          \000\004namem\000\000\000\003TPWh\002d\000\004nickm\000\000\000
-          \amojomboj"
+    # => "\203h\002d\000\004userh\003d\000\004bertd\000\004dictl\000\000\
+          000\002h\002d\000\004namem\000\000\000\003TPWh\002d\000\004nickm\
+          000\000\000\amojomboj"
     
     BERT.decode(bert)
     # => t[:user, {:name=>"TPW", :nick=>"mojombo"}]

data/Rakefile

@@ -11,9 +11,14 @@ begin
     gem.homepage = "http://github.com/mojombo/bert"
     gem.authors = ["Tom Preston-Werner"]
     gem.add_development_dependency("thoughtbot-shoulda")
-    gem.require_paths = ["lib", "ext"]
-    gem.files.include("ext")
-    gem.extensions << 'ext/bert/c/extconf.rb'
+    if ENV["JAVA"]
+      gem.extensions = nil
+      gem.platform = 'java'
+    else
+      gem.require_paths = ["lib", "ext"]
+      gem.files.include("ext")
+      gem.extensions << 'ext/bert/c/extconf.rb'
+    end
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
 rescue LoadError

data/VERSION

@@ -1 +1 @@
-1.1.2
+1.1.6

data/bert.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bert}
-  s.version = "1.1.2"
+  s.version = "1.1.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Tom Preston-Werner"]

data/ext/bert/c/decode.c

@@ -1,11 +1,9 @@
 #include "ruby.h"
-#include <string.h>
+#include <stdint.h>
+#include <netinet/in.h>
 
-#define ERL_VERSION       131
 #define ERL_SMALL_INT     97
 #define ERL_INT           98
-#define ERL_SMALL_BIGNUM  110
-#define ERL_LARGE_BIGNUM  111
 #define ERL_FLOAT         99
 #define ERL_ATOM          100
 #define ERL_SMALL_TUPLE   104
@@ -14,425 +12,485 @@
 #define ERL_STRING        107
 #define ERL_LIST          108
 #define ERL_BIN           109
+#define ERL_SMALL_BIGNUM  110
+#define ERL_LARGE_BIGNUM  111
+#define ERL_VERSION       131
 
-static VALUE mBERT;
-static VALUE cDecode;
-static VALUE cTuple;
-void Init_decode();
-
-VALUE method_decode(VALUE klass, VALUE rString);
-
-VALUE read_any_raw(unsigned char **pData);
-
-// printers
-
-void p(VALUE val) {
-  rb_funcall(rb_mKernel, rb_intern("p"), 1, val);
+#define BERT_VALID_TYPE(t) ((t) >= ERL_SMALL_INT && (t) <= ERL_LARGE_BIGNUM)
+#define BERT_TYPE_OFFSET (ERL_SMALL_INT)
+
+static VALUE rb_mBERT;
+static VALUE rb_cDecode;
+static VALUE rb_cTuple;
+
+struct bert_buf {
+	const uint8_t *data;
+	const uint8_t *end;
+};
+
+static VALUE bert_read_invalid(struct bert_buf *buf);
+
+static VALUE bert_read_sint(struct bert_buf *buf);
+static VALUE bert_read_int(struct bert_buf *buf);
+static VALUE bert_read_float(struct bert_buf *buf);
+static VALUE bert_read_atom(struct bert_buf *buf);
+static VALUE bert_read_stuple(struct bert_buf *buf);
+static VALUE bert_read_ltuple(struct bert_buf *buf);
+static VALUE bert_read_nil(struct bert_buf *buf);
+static VALUE bert_read_string(struct bert_buf *buf);
+static VALUE bert_read_list(struct bert_buf *buf);
+static VALUE bert_read_bin(struct bert_buf *buf);
+static VALUE bert_read_sbignum(struct bert_buf *buf);
+static VALUE bert_read_lbignum(struct bert_buf *buf);
+
+typedef VALUE (*bert_ptr)(struct bert_buf *buf);
+static bert_ptr bert_callbacks[] = {
+	&bert_read_sint,
+	&bert_read_int,
+	&bert_read_float,
+	&bert_read_atom,
+	&bert_read_invalid,
+	&bert_read_invalid,
+	&bert_read_invalid,
+	&bert_read_stuple,
+	&bert_read_ltuple,
+	&bert_read_nil,
+	&bert_read_string,
+	&bert_read_list,
+	&bert_read_bin,
+	&bert_read_sbignum,
+	&bert_read_lbignum
+};
+
+static inline uint8_t bert_buf_read8(struct bert_buf *buf)
+{
+	return *buf->data++;
 }
 
-// checkers
+static inline uint16_t bert_buf_read16(struct bert_buf *buf)
+{
+	/* Note that this will trigger -Wcast-align and throw a
+	 * bus error on platforms where unaligned reads are not
+	 * allowed. Also note that this is not breaking any
+	 * strict aliasing rules. */
+	uint16_t short_val = *(uint16_t *)buf->data;
+	buf->data += sizeof(uint16_t);
+	return ntohs(short_val);
+}
 
-void check_int(int num) {
-  char buf[17];
-  sprintf(buf, "%u", num);
-  rb_raise(rb_eStandardError, buf);
+static inline uint32_t bert_buf_read32(struct bert_buf *buf)
+{
+	/* Note that this will trigger -Wcast-align and throw a
+	 * bus error on platforms where unaligned reads are not
+	 * allowed. Also note that this is not breaking any
+	 * strict aliasing rules. */
+	uint32_t long_val = *(uint32_t *)buf->data;
+	buf->data += sizeof(uint32_t);
+	return ntohl(long_val);
 }
 
-void check_str(char *str) {
-  rb_raise(rb_eStandardError, str);
+static inline void bert_buf_ensure(struct bert_buf *buf, size_t size)
+{
+	if (buf->data + size > buf->end)
+		rb_raise(rb_eEOFError, "Unexpected end of BERT stream");
 }
 
-// string peekers/readers
+static VALUE bert_read(struct bert_buf *buf)
+{
+	uint8_t type;
 
-unsigned int peek_1(unsigned char **pData) {
-  return (unsigned int) **pData;
-}
+	bert_buf_ensure(buf, 1);
+	type = bert_buf_read8(buf);
 
-unsigned int peek_2(unsigned char **pData) {
-  return (unsigned int) ((**pData << 8) + *(*pData + 1));
-}
+	if (!BERT_VALID_TYPE(type))
+		rb_raise(rb_eRuntimeError, "Invalid tag '%d' for term", type);
 
-unsigned int peek_4(unsigned char **pData) {
-  return (unsigned int) ((**pData << 24) + (*(*pData + 1) << 16) + (*(*pData + 2) << 8) + *(*pData + 3));
+	return bert_callbacks[type - BERT_TYPE_OFFSET](buf);
 }
 
-unsigned int read_1(unsigned char **pData) {
-  unsigned int val = peek_1(pData);
-  *pData += 1;
-  return val;
-}
+static VALUE bert_read_dict(struct bert_buf *buf)
+{
+	uint8_t type;
+	uint32_t length = 0, i;
+	VALUE rb_dict;
 
-unsigned int read_2(unsigned char **pData) {
-  unsigned int val = peek_2(pData);
-  *pData += 2;
-  return val;
-}
+	bert_buf_ensure(buf, 1);
+	type = bert_buf_read8(buf);
 
-unsigned int read_4(unsigned char **pData) {
-  unsigned int val = peek_4(pData);
-  *pData += 4;
-  return val;
-}
+	if (type != ERL_LIST && type != ERL_NIL)
+		rb_raise(rb_eTypeError, "Invalid dict spec, not an erlang list");
 
-// tuples
+	if (type == ERL_LIST) {
+		bert_buf_ensure(buf, 4);
+		length = bert_buf_read32(buf);
+	}
 
-VALUE read_tuple(unsigned char **pData, unsigned int arity);
+	rb_dict = rb_hash_new();
 
-VALUE read_dict_pair(unsigned char **pData) {
-  if(read_1(pData) != ERL_SMALL_TUPLE) {
-    rb_raise(rb_eStandardError, "Invalid dict pair, not a small tuple");
-  }
+	for (i = 0; i < length; ++i) {
+		VALUE key, val;
+		bert_buf_ensure(buf, 2);
 
-  int arity = read_1(pData);
+		if (bert_buf_read8(buf) != ERL_SMALL_TUPLE || bert_buf_read8(buf) != 2)
+			rb_raise(rb_eTypeError, "Invalid dict tuple");
 
-  if(arity != 2) {
-    rb_raise(rb_eStandardError, "Invalid dict pair, not a 2-tuple");
-  }
+		key = bert_read(buf);
+		val = bert_read(buf);
 
-  return read_tuple(pData, arity);
-}
+		rb_hash_aset(rb_dict, key, val);
+	}
 
-VALUE read_dict(unsigned char **pData) {
-  int type = read_1(pData);
-  if(!(type == ERL_LIST || type == ERL_NIL)) {
-    rb_raise(rb_eStandardError, "Invalid dict spec, not an erlang list");
-  }
-
-  unsigned int length = 0;
-  if(type == ERL_LIST) {
-    length = read_4(pData);
-  }
-
-  VALUE cHash = rb_const_get(rb_cObject, rb_intern("Hash"));
-  VALUE hash = rb_funcall(cHash, rb_intern("new"), 0);
-
-  int i;
-  for(i = 0; i < length; ++i) {
-    VALUE pair = read_dict_pair(pData);
-    VALUE first = rb_ary_entry(pair, 0);
-    VALUE last = rb_ary_entry(pair, 1);
-    rb_funcall(hash, rb_intern("store"), 2, first, last);
-  }
-
-  if(type == ERL_LIST) {
-    read_1(pData);
-  }
-
-  return hash;
-}
+	if (type == ERL_LIST) {
+		/* disregard tail; adquire women */
+		bert_buf_ensure(buf, 1);
+		(void)bert_buf_read8(buf);
+	}
 
-VALUE read_complex_type(unsigned char **pData, int arity) {
-  VALUE type = read_any_raw(pData);
-  ID id = SYM2ID(type);
-  if(id == rb_intern("nil")) {
-    return Qnil;
-  } else if(id == rb_intern("true")) {
-    return Qtrue;
-  } else if(id == rb_intern("false")) {
-    return Qfalse;
-  } else if(id == rb_intern("time")) {
-    VALUE megasecs = read_any_raw(pData);
-    VALUE msecs = rb_funcall(megasecs, rb_intern("*"), 1, INT2NUM(1000000));
-    VALUE secs = read_any_raw(pData);
-    VALUE microsecs = read_any_raw(pData);
-    VALUE stamp = rb_funcall(msecs, rb_intern("+"), 1, secs);
-    return rb_funcall(rb_cTime, rb_intern("at"), 2, stamp, microsecs);
-  } else if(id == rb_intern("regex")) {
-    VALUE source = read_any_raw(pData);
-    VALUE opts = read_any_raw(pData);
-    int flags = 0;
-    if(rb_ary_includes(opts, ID2SYM(rb_intern("caseless"))))
-      flags = flags | 1;
-    if(rb_ary_includes(opts, ID2SYM(rb_intern("extended"))))
-      flags = flags | 2;
-    if(rb_ary_includes(opts, ID2SYM(rb_intern("multiline"))))
-      flags = flags | 4;
-    return rb_funcall(rb_cRegexp, rb_intern("new"), 2, source, INT2NUM(flags));
-  } else if(id == rb_intern("dict")) {
-    return read_dict(pData);
-  } else {
-    return Qnil;
-  }
+	return rb_dict;
 }
 
-VALUE read_tuple(unsigned char **pData, unsigned int arity) {
-  if(arity > 0) {
-    VALUE tag = read_any_raw(pData);
-    if(SYM2ID(tag) == rb_intern("bert")) {
-      return read_complex_type(pData, arity);
-    } else {
-      VALUE tuple = rb_funcall(cTuple, rb_intern("new"), 1, INT2NUM(arity));
-      rb_ary_store(tuple, 0, tag);
-      int i;
-      for(i = 1; i < arity; ++i) {
-        rb_ary_store(tuple, i, read_any_raw(pData));
-      }
-      return tuple;
-    }
-  } else {
-    return rb_funcall(cTuple, rb_intern("new"), 0);
-  }
+static inline void bert_ensure_arity(uint32_t arity, uint32_t expected)
+{
+	if (arity != expected)
+		rb_raise(rb_eTypeError, "Invalid tuple arity for complex type");
 }
 
-VALUE read_small_tuple(unsigned char **pData) {
-  if(read_1(pData) != ERL_SMALL_TUPLE) {
-    rb_raise(rb_eStandardError, "Invalid Type, not a small tuple");
-  }
+static VALUE bert_read_complex(struct bert_buf *buf, uint32_t arity)
+{
+	VALUE rb_type;
+	ID id_type;
 
-  int arity = read_1(pData);
-  return read_tuple(pData, arity);
-}
+	rb_type = bert_read(buf);
+	Check_Type(rb_type, T_SYMBOL);
 
-VALUE read_large_tuple(unsigned char **pData) {
-  if(read_1(pData) != ERL_LARGE_TUPLE) {
-    rb_raise(rb_eStandardError, "Invalid Type, not a large tuple");
-  }
+	id_type = SYM2ID(rb_type);
 
-  unsigned int arity = read_4(pData);
-  return read_tuple(pData, arity);
-}
+	if (id_type == rb_intern("nil")) {
+		bert_ensure_arity(arity, 2);
+		return Qnil;
 
-// lists
+	} else if (id_type == rb_intern("true")) {
+		bert_ensure_arity(arity, 2);
+		return Qtrue;
 
-VALUE read_list(unsigned char **pData) {
-  if(read_1(pData) != ERL_LIST) {
-    rb_raise(rb_eStandardError, "Invalid Type, not an erlang list");
-  }
+	} else if (id_type == rb_intern("false")) {
+		bert_ensure_arity(arity, 2);
+		return Qfalse;
 
-  unsigned int size = read_4(pData);
+	} else if (id_type == rb_intern("time")) {
+		VALUE rb_megasecs, rb_secs, rb_microsecs, rb_stamp, rb_msecs;
 
-  VALUE array = rb_ary_new2(size);
+		bert_ensure_arity(arity, 5);
 
-  int i;
-  for(i = 0; i < size; ++i) {
-    rb_ary_store(array, i, read_any_raw(pData));
-  }
+		rb_megasecs = bert_read(buf);
+		rb_secs = bert_read(buf);
+		rb_microsecs = bert_read(buf);
 
-  read_1(pData);
+		rb_msecs = rb_funcall(rb_megasecs, rb_intern("*"), 1, INT2NUM(1000000));
+		rb_stamp = rb_funcall(rb_msecs, rb_intern("+"), 1, rb_secs);
 
-  return array;
-}
+		return rb_funcall(rb_cTime, rb_intern("at"), 2, rb_stamp, rb_microsecs);
 
-// primitives
+	} else if (id_type == rb_intern("regex")) {
+		VALUE rb_source, rb_opts;
+		int flags = 0;
+		
+		bert_ensure_arity(arity, 4);
 
-void read_string_raw(unsigned char *dest, unsigned char **pData, unsigned int length) {
-  memcpy((char *) dest, (char *) *pData, length);
-  *(dest + length) = (unsigned char) 0;
-  *pData += length;
-}
-
-VALUE read_bin(unsigned char **pData) {
-  if(read_1(pData) != ERL_BIN) {
-    rb_raise(rb_eStandardError, "Invalid Type, not an erlang binary");
-  }
+		rb_source = bert_read(buf);
+		rb_opts = bert_read(buf);
 
-  unsigned int length = read_4(pData);
+		Check_Type(rb_source, T_STRING);
+		Check_Type(rb_opts, T_ARRAY);
 
-  VALUE rStr = rb_str_new((char *) *pData, length);
-  *pData += length;
+		if (rb_ary_includes(rb_opts, ID2SYM(rb_intern("caseless"))))
+			flags = flags | 1;
 
-  return rStr;
-}
+		if (rb_ary_includes(rb_opts, ID2SYM(rb_intern("extended"))))
+			flags = flags | 2;
 
-VALUE read_string(unsigned char **pData) {
-  if(read_1(pData) != ERL_STRING) {
-    rb_raise(rb_eStandardError, "Invalid Type, not an erlang string");
-  }
+		if (rb_ary_includes(rb_opts, ID2SYM(rb_intern("multiline"))))
+			flags = flags | 4;
 
-  int length = read_2(pData);
-  VALUE array = rb_ary_new2(length);
+		return rb_funcall(rb_cRegexp, rb_intern("new"), 2, rb_source, INT2NUM(flags));
 
-  int i = 0;
-  for(i; i < length; ++i) {
-    rb_ary_store(array, i, INT2NUM(**pData));
-    *pData += 1;
-  }
+	} else if (id_type == rb_intern("dict")) {
+		bert_ensure_arity(arity, 3);
+		return bert_read_dict(buf);
+	}
 
-  return array;
+	rb_raise(rb_eTypeError, "Invalid tag for complex value");
+	return Qnil;
 }
 
-VALUE read_atom(unsigned char **pData) {
-  if(read_1(pData) != ERL_ATOM) {
-    rb_raise(rb_eStandardError, "Invalid Type, not an atom");
-  }
+static VALUE bert_read_tuple(struct bert_buf *buf, uint32_t arity)
+{
+	if (arity > 0) {
+		VALUE rb_tag = bert_read(buf);
 
-  int length = read_2(pData);
+		if (TYPE(rb_tag) == T_SYMBOL && SYM2ID(rb_tag) == rb_intern("bert")) {
+			return bert_read_complex(buf, arity);
+		} else {
+			uint32_t i;
+			VALUE rb_tuple;
 
-  unsigned char buf[length + 1];
-  read_string_raw(buf, pData, length);
+			rb_tuple = rb_funcall(rb_cTuple, rb_intern("new"), 1, INT2NUM(arity));
+			rb_ary_store(rb_tuple, 0, rb_tag);
 
-  return ID2SYM(rb_intern((char *) buf));
-}
+			for(i = 1; i < arity; ++i)
+				rb_ary_store(rb_tuple, i, bert_read(buf));
+
+			return rb_tuple;
+		}
+	}
 
-VALUE read_small_int(unsigned char **pData) {
-  if(read_1(pData) != ERL_SMALL_INT) {
-    rb_raise(rb_eStandardError, "Invalid Type, not a small int");
-  }
+	return rb_funcall(rb_cTuple, rb_intern("new"), 0);
+}
 
-  int value = read_1(pData);
+static VALUE bert_read_stuple(struct bert_buf *buf)
+{
+	bert_buf_ensure(buf, 1);
+	return bert_read_tuple(buf, bert_buf_read8(buf));
+}
 
-  return INT2FIX(value);
+static VALUE bert_read_ltuple(struct bert_buf *buf)
+{
+	bert_buf_ensure(buf, 4);
+	return bert_read_tuple(buf, bert_buf_read32(buf));
 }
 
-VALUE read_int(unsigned char **pData) {
-  if(read_1(pData) != ERL_INT) {
-    rb_raise(rb_eStandardError, "Invalid Type, not an int");
-  }
+static VALUE bert_read_list(struct bert_buf *buf)
+{
+	uint32_t i, length;
+	VALUE rb_list;
 
-  long long value = read_4(pData);
+	bert_buf_ensure(buf, 4);
+	length = bert_buf_read32(buf);
+	rb_list = rb_ary_new2(length);
 
-  long long negative = ((value >> 31) & 0x1 == 1);
+	for(i = 0; i < length; ++i)
+		rb_ary_store(rb_list, i, bert_read(buf));
 
-  if(negative) {
-    value = (value - ((long long) 1 << 32));
-  }
+	/* disregard tail; adquire currency */
+	bert_buf_ensure(buf, 1);
+	(void)bert_buf_read8(buf);
 
-  return INT2FIX(value);
+	return rb_list;
 }
 
-VALUE read_small_bignum(unsigned char **pData) {
-  if(read_1(pData) != ERL_SMALL_BIGNUM) {
-    rb_raise(rb_eStandardError, "Invalid Type, not a small bignum");
-  }
+static VALUE bert_read_bin(struct bert_buf *buf)
+{
+	uint32_t length;
+	VALUE rb_bin;
+
+	bert_buf_ensure(buf, 4);
+	length = bert_buf_read32(buf);
 
-  unsigned int size = read_1(pData);
-  unsigned int sign = read_1(pData);
+	bert_buf_ensure(buf, length);
+	rb_bin = rb_str_new((char *)buf->data, length);
+	buf->data += length;
+
+	return rb_bin;
+}
 
-  VALUE num = INT2NUM(0);
-  VALUE tmp;
+static VALUE bert_read_string(struct bert_buf *buf)
+{
+	uint16_t i, length;
+	VALUE rb_string;
 
-  unsigned char buf[size + 1];
-  read_string_raw(buf, pData, size);
+	bert_buf_ensure(buf, 2);
+	length = bert_buf_read16(buf);
 
-  int i;
-  for(i = 0; i < size; ++i) {
-    tmp = INT2FIX(*(buf + i));
-    tmp = rb_funcall(tmp, rb_intern("<<"), 1, INT2NUM(i * 8));
-    num = rb_funcall(num, rb_intern("+"), 1, tmp);
-  }
+	bert_buf_ensure(buf, length);
+	rb_string = rb_ary_new2(length);
 
-  if(sign) {
-    num = rb_funcall(num, rb_intern("*"), 1, INT2NUM(-1));
-  }
+	for (i = 0; i < length; ++i)
+		rb_ary_store(rb_string, i, INT2FIX(buf->data[i]));
 
-  return num;
+	buf->data += length;
+	return rb_string;
 }
 
-VALUE read_large_bignum(unsigned char **pData) {
-  if(read_1(pData) != ERL_LARGE_BIGNUM) {
-    rb_raise(rb_eStandardError, "Invalid Type, not a small bignum");
-  }
+static VALUE bert_read_atom(struct bert_buf *buf)
+{
+	VALUE rb_atom;
+	uint32_t atom_len;
 
-  unsigned int size = read_4(pData);
-  unsigned int sign = read_1(pData);
+	bert_buf_ensure(buf, 2);
+	atom_len = bert_buf_read16(buf);
 
-  VALUE num = INT2NUM(0);
-  VALUE tmp;
+	/* Instead of trying to build the symbol
+	 * from here, just create a Ruby string
+	 * and internalize it. this will be faster for
+	 * unique symbols */
+	bert_buf_ensure(buf, atom_len);
+	rb_atom = rb_str_new((char *)buf->data, atom_len);
+	buf->data += atom_len;
 
-  unsigned char buf[size + 1];
-  read_string_raw(buf, pData, size);
+	return rb_str_intern(rb_atom);
+}
+
+static VALUE bert_read_sint(struct bert_buf *buf)
+{
+	bert_buf_ensure(buf, 1);
+	return INT2FIX((uint8_t)bert_buf_read8(buf));
+}
+
+static VALUE bert_read_int(struct bert_buf *buf)
+{
+	bert_buf_ensure(buf, 4);
+	return LONG2NUM((int32_t)bert_buf_read32(buf));
+}
 
-  int i;
-  for(i = 0; i < size; ++i) {
-    tmp = INT2FIX(*(buf + i));
-    tmp = rb_funcall(tmp, rb_intern("<<"), 1, INT2NUM(i * 8));
+static VALUE bert_buf_tobignum(struct bert_buf *buf, uint8_t sign, uint32_t bin_digits)
+{
+#ifdef BERT_FAST_BIGNUM
+	uint32_t *bin_buf = NULL;
+	VALUE rb_num;
+	uint32_t round_size;
+
+	bert_buf_ensure(buf, bin_digits);
+
+	/* Hack: ensure that we have at least a full word
+	 * of extra padding for the actual string, so Ruby
+	 * cannot guess the sign of the bigint from the MSB */
+	round_size = 4 + ((bin_digits + 3) & ~3);
+	bin_buf = xmalloc(round_size);
+
+	memcpy(bin_buf, buf->data, bin_digits);
+	memset((char *)bin_buf + bin_digits, 0x0, round_size - bin_digits);
+
+	/* Make Ruby unpack the string internally.
+	 * this is significantly faster than adding
+	 * the bytes one by one */
+	rb_num = rb_big_unpack(bin_buf, round_size / 4);
+
+	/* Enfore sign. So fast! */
+	RBIGNUM_SET_SIGN(rb_num, !sign);
+
+	free(bin_buf);
+	return rb_num;
+#else
+	/**
+	 * Slower bignum serialization; convert to a base16
+	 * string and then let ruby parse it internally.
+	 *
+	 * We're shipping with this by default because
+	 * `rb_big_unpack` is not trustworthy
+	 */
+	static const char to_hex[] = "0123456789abcdef";
+	char *num_str = NULL, *ptr;
+	VALUE rb_num;
+	int32_t i;
+
+	bert_buf_ensure(buf, bin_digits);
+
+	/* 2 digits per byte + sign + trailing null */
+	num_str = ptr = xmalloc((bin_digits * 2) + 2);
+
+	*ptr++ = sign ? '-' : '+';
+
+	for (i = (int32_t)bin_digits - 1; i >= 0; --i) {
+		uint8_t val = buf->data[i];
+		*ptr++ = to_hex[val >> 4];
+		*ptr++ = to_hex[val & 0xf];
+	}
+
+	*ptr = 0;
+	buf->data += bin_digits;
+
+	rb_num = rb_cstr_to_inum(num_str, 16, 1);
+	free(num_str);
+
+	return rb_num;
+#endif
+}
+
+VALUE bert_read_sbignum(struct bert_buf *buf)
+{
+	uint8_t sign, bin_digits;
 
-    num = rb_funcall(num, rb_intern("+"), 1, tmp);
-  }
+	bert_buf_ensure(buf, 2);
 
-  if(sign) {
-    num = rb_funcall(num, rb_intern("*"), 1, INT2NUM(-1));
-  }
+	bin_digits = bert_buf_read8(buf);
+	sign = bert_buf_read8(buf);
 
-  return num;
+	return bert_buf_tobignum(buf, sign, (uint32_t)bin_digits);
 }
 
-VALUE read_float(unsigned char **pData) {
-  if(read_1(pData) != ERL_FLOAT) {
-    rb_raise(rb_eStandardError, "Invalid Type, not a float");
-  }
+VALUE bert_read_lbignum(struct bert_buf *buf)
+{
+	uint32_t bin_digits;
+	uint8_t sign;
 
-  unsigned char buf[32];
-  read_string_raw(buf, pData, 31);
+	bert_buf_ensure(buf, 5);
 
-  VALUE rString = rb_str_new2((char *) buf);
+	bin_digits = bert_buf_read32(buf);
+	sign = bert_buf_read8(buf);
 
-  return rb_funcall(rString, rb_intern("to_f"), 0);
+	return bert_buf_tobignum(buf, sign, bin_digits);
 }
 
-VALUE read_nil(unsigned char **pData) {
-  if(read_1(pData) != ERL_NIL) {
-    rb_raise(rb_eStandardError, "Invalid Type, not a nil list");
-  }
+/*
+ * -------------------
+ * |1  | 31          |
+ * |99 | Float String|
+ * -------------------
+ *
+ * A float is stored in string format. the format used in sprintf
+ * to format the float is "%.20e" (there are more bytes allocated 
+ * than necessary). To unpack the float use sscanf with format "%lf".
+ */
+static VALUE bert_read_float(struct bert_buf *buf)
+{
+	VALUE rb_float;
+
+	bert_buf_ensure(buf, 31);
+
+	rb_float = rb_str_new((char *)buf->data, 31);
+	buf->data += 31;
+
+	return rb_funcall(rb_float, rb_intern("to_f"), 0);
+}
 
-  return rb_ary_new2(0);
+static VALUE bert_read_nil(struct bert_buf *buf)
+{
+	return rb_ary_new2(0);
 }
 
-// read_any_raw
-
-VALUE read_any_raw(unsigned char **pData) {
-  switch(peek_1(pData)) {
-    case ERL_SMALL_INT:
-      return read_small_int(pData);
-      break;
-    case ERL_INT:
-      return read_int(pData);
-      break;
-    case ERL_FLOAT:
-      return read_float(pData);
-      break;
-    case ERL_ATOM:
-      return read_atom(pData);
-      break;
-    case ERL_SMALL_TUPLE:
-      return read_small_tuple(pData);
-      break;
-    case ERL_LARGE_TUPLE:
-      return read_large_tuple(pData);
-      break;
-    case ERL_NIL:
-      return read_nil(pData);
-      break;
-    case ERL_STRING:
-      return read_string(pData);
-      break;
-    case ERL_LIST:
-      return read_list(pData);
-      break;
-    case ERL_BIN:
-      return read_bin(pData);
-      break;
-    case ERL_SMALL_BIGNUM:
-      return read_small_bignum(pData);
-      break;
-    case ERL_LARGE_BIGNUM:
-      return read_large_bignum(pData);
-      break;
-  }
-  return Qnil;
+static VALUE bert_read_invalid(struct bert_buf *buf)
+{
+	rb_raise(rb_eTypeError, "Invalid object tag in BERT stream");
+	return Qnil;
 }
 
-VALUE method_decode(VALUE klass, VALUE rString) {
-  unsigned char *data = (unsigned char *) StringValuePtr(rString);
+static VALUE rb_bert_decode(VALUE klass, VALUE rb_string)
+{
+	struct bert_buf buf;
 
-  unsigned char **pData = &data;
+	Check_Type(rb_string, T_STRING);
+	buf.data = (uint8_t *)RSTRING_PTR(rb_string);
+	buf.end = buf.data + RSTRING_LEN(rb_string);
 
-  // check protocol version
-  if(read_1(pData) != ERL_VERSION) {
-    rb_raise(rb_eStandardError, "Bad Magic");
-  }
+	bert_buf_ensure(&buf, 1);
 
-  return read_any_raw(pData);
+	if (bert_buf_read8(&buf) != ERL_VERSION)
+		rb_raise(rb_eTypeError, "Invalid magic value for BERT string");
+
+	return bert_read(&buf);
 }
 
-VALUE method_impl(VALUE klass) {
-  return rb_str_new("C", 1);
+static VALUE rb_bert_impl(VALUE klass)
+{
+	return rb_str_new("C", 1);
 }
 
-void Init_decode() {
-  mBERT = rb_const_get(rb_cObject, rb_intern("BERT"));
-  cDecode = rb_define_class_under(mBERT, "Decode", rb_cObject);
-  cTuple = rb_const_get(mBERT, rb_intern("Tuple"));
-  rb_define_singleton_method(cDecode, "decode", method_decode, 1);
-  rb_define_singleton_method(cDecode, "impl", method_impl, 0);
+void Init_decode()
+{
+	rb_mBERT = rb_const_get(rb_cObject, rb_intern("BERT"));
+	rb_cTuple = rb_const_get(rb_mBERT, rb_intern("Tuple"));
+
+	rb_cDecode = rb_define_class_under(rb_mBERT, "Decode", rb_cObject);
+	rb_define_singleton_method(rb_cDecode, "decode", rb_bert_decode, 1);
+	rb_define_singleton_method(rb_cDecode, "impl", rb_bert_impl, 0);
 }

data/ext/bert/c/extconf.rb

@@ -1,6 +1,9 @@
 # Loads mkmf which is used to make makefiles for Ruby extensions
 require 'mkmf'
 
+# warnings save lives
+$CFLAGS << " -Wall "
+
 # Give it a name
 extension_name = 'decode'
 

data/lib/bert/decode.rb

@@ -8,7 +8,9 @@ module BERT
     end
 
     def self.decode(string)
-      new(StringIO.new(string)).read_any
+      io = StringIO.new(string)
+      io.set_encoding('binary') if io.respond_to?(:set_encoding)
+      new(io).read_any
     end
 
     def initialize(ins)
@@ -113,7 +115,7 @@ module BERT
       value = read_4
       negative = (value >> 31)[0] == 1
       value = (value - (1 << 32)) if negative
-      value = Fixnum.induced_from(value)
+      value
     end
 
     def read_small_bignum
@@ -126,7 +128,7 @@ module BERT
         value = (byte * (256 ** index))
         sign != 0 ? (result - value) : (result + value)
       end
-      Bignum.induced_from(added)
+      added
     end
 
     def read_large_bignum
@@ -139,7 +141,7 @@ module BERT
         value = (byte * (256 ** index))
         sign != 0 ? (result - value) : (result + value)
       end
-      Bignum.induced_from(added)
+      added
     end
 
     def read_float

data/lib/bert/encode.rb

@@ -10,6 +10,7 @@ module BERT
 
     def self.encode(data)
       io = StringIO.new
+      io.set_encoding('binary') if io.respond_to?(:set_encoding)
       self.new(io).write_any(data)
       io.string
     end
@@ -56,7 +57,7 @@ module BERT
       fail(sym) unless sym.is_a?(Symbol)
       data = sym.to_s
       write_1 ATOM
-      write_2 data.length
+      write_2 data.bytesize
       write_string data
     end
 
@@ -78,7 +79,7 @@ module BERT
     end
 
     def write_bignum(num)
-      n = (num.to_s(2).size / 8.0).ceil
+      n = (num.abs.to_s(2).size / 8.0).ceil
       if n < 256
         write_1 SMALL_BIGNUM
         write_1 n
@@ -125,7 +126,7 @@ module BERT
 
     def write_binary(data)
       write_1 BIN
-      write_4 data.length
+      write_4 data.bytesize
       write_string data
     end
 

data/test/bert_test.rb

@@ -44,7 +44,8 @@ class BertTest < Test::Unit::TestCase
       dd << {:a => 1, :b => 2}
       dd << Time.now
       dd << /^c(a)t$/i
-      
+
+      dd << 178
       dd << 256**256 - 1
 
       dd << :true

data/test/encoder_test.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'test_helper'
 
 class EncoderTest < Test::Unit::TestCase
@@ -79,9 +81,22 @@ class EncoderTest < Test::Unit::TestCase
       assert cruby[1].instance_of?(BERT::Tuple)
     end
 
+    should 'handle utf8 strings' do
+      bert = [131, 109, 0, 0, 0, 5, 195, 169, 116, 195, 169].pack('C*')
+      assert_equal bert, BERT::Encoder.encode("été")
+    end
+
+    should 'handle utf8 symbols' do
+      bert = [131, 100, 0, 5, 195, 169, 116, 195, 169].pack('C*')
+      assert_equal bert, BERT::Encoder.encode(:'été')
+    end
+
     should "handle bignums" do
       bert = [131,110,8,0,0,0,232,137,4,35,199,138].pack('c*')
       assert_equal bert, BERT::Encoder.encode(10_000_000_000_000_000_000)
+
+      bert = [131,110,8,1,0,0,232,137,4,35,199,138].pack('c*')
+      assert_equal bert, BERT::Encoder.encode(-10_000_000_000_000_000_000)
     end
 
     should "leave other stuff alone" do

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: bert
 version: !ruby/object:Gem::Version 
-  version: 1.1.2
+  hash: 31
+  prerelease: 
+  segments: 
+  - 1
+  - 1
+  - 6
+  version: 1.1.6
 platform: ruby
 authors: 
 - Tom Preston-Werner
@@ -14,14 +20,18 @@ default_executable:
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: thoughtbot-shoulda
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :development
+  version_requirements: *id001
 description: BERT Serializiation for Ruby
 email: tom@mojombo.com
 executables: []
@@ -69,21 +79,27 @@ require_paths:
 - lib
 - ext
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: BERT Serializiation for Ruby