lwes 0.3.1 → 0.4.0

data/ChangeLog

@@ -1,3 +1,8 @@
+Version 0.4.0 (erik-s-chang)
+  * large performance improvements for ESF + Struct users with large events,
+    one real world app is nearly twice as fast
+  * GC safety fixes when raising exceptions
+
 Version 0.3.1 (erik-s-chang)
   * fix tests and enable Ruby warnings for tests
 

data/ext/lwes/emitter.c

@@ -1,14 +1,70 @@
 #include "lwes_ruby.h"
 
-static VALUE cLWES_Emitter;
-static ID id_TYPE_DB, id_TYPE_LIST, id_NAME;
-static ID id_new;
+static VALUE ENC; /* LWES_ENCODING */
+static ID id_TYPE_DB, id_TYPE_LIST, id_NAME, id_HAVE_ENCODING;
+static ID id_new, id_enc, id_size;
+static ID sym_enc;
+
+static void dump_name(VALUE name, LWES_BYTE_P buf, size_t *off)
+{
+	char *s = RSTRING_PTR(name);
+
+	if (marshall_SHORT_STRING(s, buf, MAX_MSG_SIZE, off) > 0)
+		return;
+	rb_raise(rb_eRuntimeError, "failed to dump name=%s", s);
+}
+
+static int dump_bool(VALUE name, VALUE val, LWES_BYTE_P buf, size_t *off)
+{
+	LWES_BOOLEAN tmp = FALSE;
+
+	if (val == Qtrue) {
+		tmp = TRUE;
+	} else if (val != Qfalse) {
+		volatile VALUE raise_inspect;
+
+		rb_raise(rb_eTypeError, "non-boolean set for %s: %s",
+			 RSTRING_PTR(name),
+			 RAISE_INSPECT(val));
+	}
+	dump_name(name, buf, off);
+	lwesrb_dump_type(LWES_BOOLEAN_TOKEN, buf, off);
+	return marshall_BOOLEAN(tmp, buf, MAX_MSG_SIZE, off);
+}
+
+static int dump_string(VALUE name, VALUE val, LWES_BYTE_P buf, size_t *off)
+{
+	volatile VALUE raise_inspect;
+
+	if (TYPE(val) != T_STRING)
+		rb_raise(rb_eTypeError, "not a string: %s",
+		         RAISE_INSPECT(val));
+	dump_name(name, buf, off);
+	lwesrb_dump_type(LWES_STRING_TOKEN, buf, off);
+	return marshall_LONG_STRING(RSTRING_PTR(val), buf, MAX_MSG_SIZE, off);
+}
+
+static void dump_enc(VALUE enc, LWES_BYTE_P buf, size_t *off)
+{
+	dump_name(ENC, buf, off);
+	lwesrb_dump_num(LWES_INT_16_TOKEN, enc, buf, off);
+}
+
+static char *my_strdup(const char *str)
+{
+	long len = strlen(str) + 1;
+	char *rv = xmalloc(len);
+
+	memcpy(rv, str, len);
+
+	return rv;
+}
 
 /* the underlying struct for LWES::Emitter */
 struct _rb_lwes_emitter {
 	struct lwes_emitter *emitter;
-	LWES_CONST_SHORT_STRING address;
-	LWES_CONST_SHORT_STRING iface;
+	char *address;
+	char *iface;
 	LWES_U_INT_32 port;
 	LWES_BOOLEAN emit_heartbeat;
 	LWES_INT_16 freq;
@@ -32,28 +88,11 @@ static void rle_free(void *ptr)
 
 	if (rle->emitter)
 		lwes_emitter_destroy(rle->emitter);
+	xfree(rle->address);
+	xfree(rle->iface);
 	xfree(ptr);
 }
 
-static struct lwes_event *
-lwesrb_event_create(struct lwes_event_type_db *db, VALUE name)
-{
-	int gc_retry = 1;
-	const char *event_name = RSTRING_PTR(name);
-	struct lwes_event *event;
-
-retry:
-	event = lwes_event_create(db, event_name);
-	if (!event) {
-		if (--gc_retry == 0) {
-			rb_gc();
-			goto retry;
-		}
-		rb_raise(rb_eRuntimeError, "failed to create lwes_event");
-	}
-	return event;
-}
-
 /* called by the GC when object is allocated */
 static VALUE rle_alloc(VALUE klass)
 {
@@ -73,202 +112,213 @@ static VALUE rle_alloc(VALUE klass)
  */
 static VALUE event_hash_iter_i(VALUE kv, VALUE memo)
 {
-	VALUE *tmp;
+	volatile VALUE raise_inspect;
+	VALUE *tmp = (VALUE *)memo;
 	VALUE val;
-	struct lwes_event *event = (struct lwes_event *)memo;
-	LWES_CONST_SHORT_STRING name;
+	VALUE name;
 	int rv = 0;
-	int gc_retry = 1;
+	LWES_BYTE_P buf = (LWES_BYTE_P)tmp[0];
+	size_t *off = (size_t *)tmp[1];
 
-	assert(TYPE(kv) == T_ARRAY &&
-	       "hash iteration not giving key-value pairs");
+	if (TYPE(kv) != T_ARRAY || RARRAY_LEN(kv) != 2)
+		rb_raise(rb_eTypeError,
+		         "hash iteration not giving key-value pairs");
 	tmp = RARRAY_PTR(kv);
-	name = RSTRING_PTR(rb_obj_as_string(tmp[0]));
+	name = tmp[0];
+
+	if (name == sym_enc) return Qnil; /* already dumped first */
+
+	name = rb_obj_as_string(name);
+
+	if (strcmp(RSTRING_PTR(name), LWES_ENCODING) == 0)
+		return Qnil;
+
 	val = tmp[1];
 
-retry:
 	switch (TYPE(val)) {
 	case T_TRUE:
-		rv = lwes_event_set_BOOLEAN(event, name, TRUE);
-		break;
 	case T_FALSE:
-		rv = lwes_event_set_BOOLEAN(event, name, FALSE);
+		rv = dump_bool(name, val, buf, off);
 		break;
 	case T_ARRAY:
-		rv = lwesrb_event_set_numeric(event, name, val);
-		break;
+		dump_name(name, buf, off);
+		lwesrb_dump_num_ary(val, buf, off);
+		return Qnil;
 	case T_STRING:
-		rv = lwes_event_set_STRING(event, name, RSTRING_PTR(val));
+		rv = dump_string(name, val, buf, off);
 		break;
 	}
-	if (rv > 0) {
+
+	if (rv > 0)
 		return Qnil;
-	} else if (rv == 0) {
-		rb_raise(rb_eArgError, "unhandled type %s=%s for event=%s",
-	                 name, RSTRING_PTR(rb_inspect(val)), event->eventName);
-	} else {
-		/* looking at the lwes source code, -3 is allocation errors */
-		if (rv == -3 && --gc_retry == 0) {
-			rb_gc();
-			goto retry;
-		}
-		rb_raise(rb_eRuntimeError, "failed to set %s=%s for event=%s",
-			 name, RSTRING_PTR(rb_inspect(val)), event->eventName);
-	}
+
+	rb_raise(rb_eArgError, "unhandled type %s=%s",
+		 RSTRING_PTR(name), RAISE_INSPECT(val));
 	return Qfalse;
 }
 
-static VALUE _emit_hash(VALUE _tmp)
+static VALUE emit_hash(VALUE self, VALUE name, VALUE event)
 {
-	VALUE *tmp = (VALUE *)_tmp;
-	VALUE self = tmp[0];
-	VALUE _event = tmp[1];
-	struct lwes_event *event = (struct lwes_event *)tmp[2];
+	struct _rb_lwes_emitter *rle = _rle(self);
+	LWES_BYTE_P buf = rle->emitter->buffer;
+	VALUE tmp[2];
+	size_t off = 0;
+	VALUE enc;
+	int size = NUM2INT(rb_funcall(event, id_size, 0, 0));
+	int rv;
+
+	tmp[0] = (VALUE)buf;
+	tmp[1] = (VALUE)&off;
+
+	if (size < 0 || size > UINT16_MAX)
+		rb_raise(rb_eRangeError, "hash size out of uint16 range");
+
+	/* event name first */
+	dump_name(name, buf, &off);
+
+	/* number of attributes second */
+	rv = marshall_U_INT_16((LWES_U_INT_16)size, buf, MAX_MSG_SIZE, &off);
+	if (rv <= 0)
+		rb_raise(rb_eRuntimeError, "failed to dump num_attrs");
+
+	/* dump encoding before other fields */
+	enc = rb_hash_aref(event, sym_enc);
+	if (NIL_P(enc))
+		enc = rb_hash_aref(event, ENC);
+	if (! NIL_P(enc))
+		dump_enc(enc, buf, &off);
 
-	rb_iterate(rb_each, _event, event_hash_iter_i, (VALUE)event);
-	if (lwes_emitter_emit(_rle(self)->emitter, event) < 0)
+	/* the rest of the fields */
+	rb_iterate(rb_each, event, event_hash_iter_i, (VALUE)&tmp);
+
+	if (lwes_emitter_emit_bytes(rle->emitter, buf, off) < 0)
 		rb_raise(rb_eRuntimeError, "failed to emit event");
 
-	return _event;
+	return event;
 }
 
 static void
-set_field(
-	struct lwes_event *event,
-	LWES_CONST_SHORT_STRING name,
+marshal_field(
+	VALUE name,
 	LWES_TYPE type,
-	VALUE val)
+	VALUE val,
+	LWES_BYTE_P buf,
+	size_t *off)
 {
-	int gc_retry = 1;
-	int rv;
+	volatile VALUE raise_inspect;
 
-retry:
 	switch (type) {
-	case LWES_TYPE_BOOLEAN:
-		if (val == Qfalse)
-			rv = lwes_event_set_BOOLEAN(event, name, FALSE);
-		else if (val == Qtrue)
-			rv = lwes_event_set_BOOLEAN(event, name, TRUE);
-		else
-			rb_raise(rb_eTypeError, "non-boolean set for %s: %s",
-			         name, RSTRING_PTR(rb_inspect(val)));
-		break;
 	case LWES_TYPE_STRING:
-		if (TYPE(val) != T_STRING)
-			rb_raise(rb_eTypeError, "non-String set for %s: %s",
-			         name, RSTRING_PTR(rb_inspect(val)));
-		rv = lwes_event_set_STRING(event, name, RSTRING_PTR(val));
+		if (dump_string(name, val, buf, off) > 0)
+			return;
+		break;
+	case LWES_TYPE_BOOLEAN:
+		if (dump_bool(name, val, buf, off) > 0)
+			return;
 		break;
 	default:
-		rv = lwesrb_event_set_num(event, name, type, val);
-	}
-	if (rv > 0) {
+		dump_name(name, buf, off);
+		lwesrb_dump_num(type, val, buf, off);
 		return;
-	} else {
-		if (rv == -3 && --gc_retry == 0) {
-			rb_gc();
-			goto retry;
-		}
-		rb_raise(rb_eRuntimeError,
-			 "failed to set %s=%s for event=%s (error: %d)",
-			 name, RSTRING_PTR(rb_inspect(val)),
-			 event->eventName, rv);
 	}
 
-	assert(0 && "you should never get here (set_field)");
+	rb_raise(rb_eRuntimeError, "failed to set %s=%s",
+		 RSTRING_PTR(name), RAISE_INSPECT(val));
 }
 
-static VALUE _emit_struct(VALUE _argv)
+static void lwes_struct_class(
+	VALUE *event_class,
+	VALUE *name,
+	VALUE *type_list,
+	VALUE *have_enc,
+	VALUE event)
 {
-	VALUE *argv = (VALUE *)_argv;
-	VALUE self = argv[0];
-	VALUE _event = argv[1];
-	struct lwes_event *event = (struct lwes_event *)argv[2];
-	VALUE type_list = argv[3];
-	long i;
-	VALUE *tmp;
-
-	if (TYPE(type_list) != T_ARRAY)
-		rb_raise(rb_eArgError, "could not get TYPE_LIST const");
+	VALUE type_db;
+	volatile VALUE raise_inspect;
 
-	i = RARRAY_LEN(type_list);
-	for (tmp = RARRAY_PTR(type_list); --i >= 0; tmp++) {
-		/* inner: [ :field_sym, "field_name", type ] */
-		VALUE *inner = RARRAY_PTR(*tmp);
-		VALUE val = rb_struct_aref(_event, inner[0]);
-		LWES_CONST_SHORT_STRING name;
-		LWES_TYPE type;
+	*event_class = CLASS_OF(event);
+	type_db = rb_const_get(*event_class, id_TYPE_DB);
 
-		if (NIL_P(val))
-			continue;
+	if (CLASS_OF(type_db) != cLWES_TypeDB)
+		rb_raise(rb_eArgError, "class does not have valid TYPE_DB");
 
-		name = RSTRING_PTR(inner[1]);
-		type = NUM2INT(inner[2]);
-		set_field(event, name, type, val);
-	}
+	*name = rb_const_get(*event_class, id_NAME);
+	if (TYPE(*name) != T_STRING)
+		rb_raise(rb_eArgError,
+		         "could not get valid const NAME: %s",
+		         RAISE_INSPECT(event));
 
-	if (lwes_emitter_emit(_rle(self)->emitter, event) < 0)
-		rb_raise(rb_eRuntimeError, "failed to emit event");
+	*type_list = rb_const_get(*event_class, id_TYPE_LIST);
+	if (TYPE(*type_list) != T_ARRAY)
+		rb_raise(rb_eArgError,
+		         "could not get valid const TYPE_LIST: %s",
+		         RAISE_INSPECT(event));
 
-	return _event;
+	*have_enc = rb_const_get(*event_class, id_HAVE_ENCODING);
 }
 
-static VALUE _destroy_event(VALUE _event)
+static VALUE emit_struct(VALUE self, VALUE event)
 {
-	struct lwes_event *event = (struct lwes_event *)_event;
-
-	assert(event && "destroying NULL event");
-	lwes_event_destroy(event);
-
-	return Qnil;
-}
+	VALUE event_class, name, type_list, have_enc;
+	struct _rb_lwes_emitter *rle = _rle(self);
+	LWES_BYTE_P buf = rle->emitter->buffer;
+	size_t off = 0;
+	long i;
+	VALUE *tmp;
+	LWES_U_INT_16 num_attr = 0;
+	size_t num_attr_off;
+	VALUE *flds;
 
-static VALUE emit_hash(VALUE self, VALUE name, VALUE _event)
-{
-	VALUE tmp[3];
-	struct lwes_event *event = lwesrb_event_create(NULL, name);
+	lwes_struct_class(&event_class, &name, &type_list, &have_enc, event);
 
-	tmp[0] = self;
-	tmp[1] = _event;
-	tmp[2] = (VALUE)event;
-	rb_ensure(_emit_hash, (VALUE)&tmp, _destroy_event, (VALUE)event);
+	/* event name */
+	dump_name(name, buf, &off);
 
-	return _event;
-}
+	/* number of attributes, use a placeholder until we've iterated */
+	num_attr_off = off;
+	if (marshall_U_INT_16(0, buf, MAX_MSG_SIZE, &off) < 0)
+		rb_raise(rb_eRuntimeError,
+		         "failed to marshal number_of_attributes");
+
+	/* dump encoding before other fields */
+	if (have_enc == Qtrue) {
+		VALUE enc = rb_funcall(event, id_enc, 0, 0);
+		if (! NIL_P(enc)) {
+			++num_attr;
+			dump_enc(enc, buf, &off);
+		}
+	}
 
-static struct lwes_event_type_db * get_type_db(VALUE event_class)
-{
-	VALUE type_db = rb_const_get(event_class, id_TYPE_DB);
+	i = RARRAY_LEN(type_list);
+	flds = RSTRUCT_PTR(event);
+	tmp = RARRAY_PTR(type_list);
+	for (; --i >= 0; tmp++, flds++) {
+		/* inner: [ :field_sym, "field_name", type ] */
+		VALUE *inner = RARRAY_PTR(*tmp);
+		VALUE val, name;
+		LWES_TYPE type;
 
-        if (CLASS_OF(type_db) != cLWES_TypeDB)
-		rb_raise(rb_eArgError, "class does not have valid TYPE_DB");
+		if (inner[0] == sym_enc) /* encoding was already dumped */
+			continue;
 
-	return lwesrb_get_type_db(type_db);
-}
+		val = *flds;
+		if (NIL_P(val))
+			continue; /* LWES doesn't know nil */
 
-static VALUE emit_struct(VALUE self, VALUE _event)
-{
-	VALUE argv[4];
-	VALUE event_class = CLASS_OF(_event);
-	struct lwes_event_type_db *db = get_type_db(event_class);
-	struct lwes_event *event;
-	VALUE name = rb_const_get(event_class, id_NAME);
-	VALUE type_list = rb_const_get(event_class, id_TYPE_LIST);
-
-	if (TYPE(name) != T_STRING || TYPE(type_list) != T_ARRAY)
-		rb_raise(rb_eArgError,
-		         "could not get class NAME or TYPE_LIST from: %s",
-		         RSTRING_PTR(rb_inspect(_event)));
+		name = inner[1];
+		type = NUM2INT(inner[2]);
+		++num_attr;
+		marshal_field(name, type, val, buf, &off);
+	}
 
-	event = lwesrb_event_create(db, name);
+	/* now we've iterated, we can accurately give num_attr */
+	if (marshall_U_INT_16(num_attr, buf, MAX_MSG_SIZE, &num_attr_off) <= 0)
+		rb_raise(rb_eRuntimeError, "failed to marshal num_attr");
 
-	argv[0] = self;
-	argv[1] = _event;
-	argv[2] = (VALUE)event;
-	argv[3] = type_list;
-	rb_ensure(_emit_struct, (VALUE)&argv, _destroy_event, (VALUE)event);
+	if (lwes_emitter_emit_bytes(rle->emitter, buf, off) < 0)
+		rb_raise(rb_eRuntimeError, "failed to emit event");
 
-	return _event;
+	return event;
 }
 
 /*
@@ -280,12 +330,12 @@ static VALUE emit_struct(VALUE self, VALUE _event)
  */
 static VALUE emitter_ltlt(VALUE self, VALUE event)
 {
-	if (TYPE(event) != T_STRUCT)
-		rb_raise(rb_eArgError,
-		         "Must be a Struct: %s",
-		         RSTRING_PTR(rb_inspect(event)));
+	volatile VALUE raise_inspect;
+
+	if (TYPE(event) == T_STRUCT)
+		return emit_struct(self, event);
 
-	return emit_struct(self, event);
+	rb_raise(rb_eArgError, "Must be a Struct: %s", RAISE_INSPECT(event));
 }
 
 /*
@@ -302,6 +352,7 @@ static VALUE emitter_ltlt(VALUE self, VALUE event)
  */
 static VALUE emitter_emit(int argc, VALUE *argv, VALUE self)
 {
+	volatile VALUE raise_inspect;
 	VALUE name = Qnil;
 	VALUE event = Qnil;
 	argc = rb_scan_args(argc, argv, "11", &name, &event);
@@ -335,7 +386,7 @@ static VALUE emitter_emit(int argc, VALUE *argv, VALUE self)
 	default:
 		rb_raise(rb_eArgError,
 		         "bad argument: %s, must be a String, Struct or Class",
-			 RSTRING_PTR(rb_inspect(name)));
+			 RAISE_INSPECT(name));
 	}
 
 	assert(0 && "should never get here");
@@ -388,6 +439,9 @@ static VALUE init_copy(VALUE dest, VALUE obj)
 	struct _rb_lwes_emitter *src = _rle(obj);
 
 	memcpy(dst, src, sizeof(*dst));
+	dst->address = my_strdup(src->address);
+	if (dst->iface)
+		dst->iface = my_strdup(src->iface);
 	lwesrb_emitter_create(dst);
 
 	assert(dst->emitter && dst->emitter != src->emitter &&
@@ -414,7 +468,7 @@ static VALUE _create(VALUE self, VALUE options)
 	address = rb_hash_aref(options, ID2SYM(rb_intern("address")));
 	if (TYPE(address) != T_STRING)
 		rb_raise(rb_eTypeError, ":address must be a string");
-	rle->address = RSTRING_PTR(address);
+	rle->address = my_strdup(RSTRING_PTR(address));
 
 	iface = rb_hash_aref(options, ID2SYM(rb_intern("iface")));
 	switch (TYPE(iface)) {
@@ -422,7 +476,7 @@ static VALUE _create(VALUE self, VALUE options)
 		rle->iface = NULL;
 		break;
 	case T_STRING:
-		rle->iface = RSTRING_PTR(iface);
+		rle->iface = my_strdup(RSTRING_PTR(iface));
 		break;
 	default:
 		rb_raise(rb_eTypeError, ":iface must be a String or nil");
@@ -463,7 +517,8 @@ static VALUE _create(VALUE self, VALUE options)
 void lwesrb_init_emitter(void)
 {
 	VALUE mLWES = rb_define_module("LWES");
-	cLWES_Emitter = rb_define_class_under(mLWES, "Emitter", rb_cObject);
+	VALUE cLWES_Emitter =
+	                  rb_define_class_under(mLWES, "Emitter", rb_cObject);
 
 	rb_define_method(cLWES_Emitter, "<<", emitter_ltlt, 1);
 	rb_define_method(cLWES_Emitter, "emit", emitter_emit, -1);
@@ -474,5 +529,12 @@ void lwesrb_init_emitter(void)
 	LWESRB_MKID(TYPE_DB);
 	LWESRB_MKID(TYPE_LIST);
 	LWESRB_MKID(NAME);
-        LWESRB_MKID(new);
+	LWESRB_MKID(HAVE_ENCODING);
+	LWESRB_MKID(new);
+	LWESRB_MKID(size);
+	id_enc = rb_intern(LWES_ENCODING);
+	sym_enc = ID2SYM(id_enc);
+
+	ENC = rb_obj_freeze(rb_str_new2(LWES_ENCODING));
+	rb_define_const(mLWES, "ENCODING", ENC);
 }

data/ext/lwes/lwes.c

@@ -1,11 +1,9 @@
 #include "lwes_ruby.h"
 
-static VALUE mLWES;
-
 /* initialize the extension, Ruby automatically picks this up */
 void Init_lwes_ext(void)
 {
-	mLWES = rb_define_module("LWES");
+	VALUE mLWES = rb_define_module("LWES");
 
 	#define LWES_TYPE_CONST(name) \
 	   rb_define_const(mLWES, #name, INT2FIX(LWES_TYPE_##name))

data/ext/lwes/lwes_ruby.h

@@ -19,16 +19,11 @@ void lwesrb_init_emitter(void);
 
 void lwesrb_init_numeric(void);
 
-int lwesrb_event_set_numeric(
-	struct lwes_event *event,
-	LWES_CONST_SHORT_STRING name,
-	VALUE array);
+void lwesrb_dump_type(LWES_BYTE type, LWES_BYTE_P buf, size_t *off);
 
-int lwesrb_event_set_num(
-	struct lwes_event *event,
-	LWES_CONST_SHORT_STRING name,
-	LWES_TYPE type,
-	VALUE val);
+void lwesrb_dump_num(LWES_BYTE type, VALUE val, LWES_BYTE_P buf, size_t *off);
+
+void lwesrb_dump_num_ary(VALUE array, LWES_BYTE_P buf, size_t *off);
 
 #ifndef RSTRING_PTR
 #  define RSTRING_PTR(s) (RSTRING(s)->ptr)
@@ -40,4 +35,11 @@ int lwesrb_event_set_num(
 #  define RARRAY_LEN(s) (RARRAY(s)->len)
 #endif
 
+#ifndef RSTRUCT_PTR
+#  define RSTRUCT_PTR(s) (RSTRUCT(s)->ptr)
+#  define RSTRUCT_LEN(s) (RSTRUCT(s)->len)
+#endif
+
+#define RAISE_INSPECT(v) RSTRING_PTR(raise_inspect = rb_inspect(v))
+
 #endif /* LWES_RUBY_H */

data/ext/lwes/numeric.c

@@ -7,8 +7,14 @@ static ID
   sym_int64, sym_uint64,
   sym_ip_addr;
 
-static int set_uint16(
-	struct lwes_event *event, LWES_CONST_SHORT_STRING name, VALUE val)
+void lwesrb_dump_type(LWES_BYTE type, LWES_BYTE_P buf, size_t *off)
+{
+	if (marshall_BYTE(type, buf, MAX_MSG_SIZE, off) > 0)
+		return;
+	rb_raise(rb_eRuntimeError, "failed to dump type=%02x", (unsigned)type);
+}
+
+static int dump_uint16(VALUE val, LWES_BYTE_P buf, size_t *off)
 {
 	int32_t tmp = NUM2INT(val);
 
@@ -17,24 +23,24 @@ static int set_uint16(
 	if (tmp > UINT16_MAX)
 		rb_raise(rb_eRangeError, ":uint16 too large: %d", tmp);
 
-	return lwes_event_set_U_INT_16(event, name, (LWES_U_INT_16)tmp);
+	lwesrb_dump_type(LWES_U_INT_16_TOKEN, buf, off);
+	return marshall_U_INT_16((LWES_U_INT_16)tmp, buf, MAX_MSG_SIZE, off);
 }
 
-static int set_int16(
-	struct lwes_event *event, LWES_CONST_SHORT_STRING name, VALUE val)
+static int dump_int16(VALUE val, LWES_BYTE_P buf, size_t *off)
 {
-	int tmp = NUM2INT(val);
+	int32_t tmp = NUM2INT(val);
 
 	if (tmp > INT16_MAX)
 		rb_raise(rb_eRangeError, ":int16 too large: %i", tmp);
 	if (tmp < INT16_MIN)
 		rb_raise(rb_eRangeError, ":int16 too small: %i", tmp);
 
-	return lwes_event_set_INT_16(event, name, (LWES_INT_16)tmp);
+	lwesrb_dump_type(LWES_INT_16_TOKEN, buf, off);
+	return marshall_INT_16((LWES_INT_16)tmp, buf, MAX_MSG_SIZE, off);
 }
 
-static int set_uint32(
-	struct lwes_event *event, LWES_CONST_SHORT_STRING name, VALUE val)
+static int dump_uint32(VALUE val, LWES_BYTE_P buf, size_t *off)
 {
 	LONG_LONG tmp = NUM2LL(val);
 
@@ -43,11 +49,11 @@ static int set_uint32(
 	if (tmp > UINT32_MAX)
 		rb_raise(rb_eRangeError, ":uint32 too large: %lli", tmp);
 
-	return lwes_event_set_U_INT_32(event, name, (LWES_U_INT_32)tmp);
+	lwesrb_dump_type(LWES_U_INT_32_TOKEN, buf, off);
+	return marshall_U_INT_32((LWES_U_INT_32)tmp, buf, MAX_MSG_SIZE, off);
 }
 
-static int set_int32(
-	struct lwes_event *event, LWES_CONST_SHORT_STRING name, VALUE val)
+static int dump_int32(VALUE val, LWES_BYTE_P buf, size_t *off)
 {
 	LONG_LONG tmp = NUM2LL(val);
 
@@ -56,66 +62,63 @@ static int set_int32(
 	if (tmp < INT32_MIN)
 		rb_raise(rb_eRangeError, ":int32 too small: %lli", tmp);
 
-	return lwes_event_set_INT_32(event, name, (LWES_INT_32)tmp);
+	lwesrb_dump_type(LWES_INT_32_TOKEN, buf, off);
+	return marshall_INT_32((LWES_INT_32)tmp, buf, MAX_MSG_SIZE, off);
 }
 
-static int set_uint64(
-	struct lwes_event *event, LWES_CONST_SHORT_STRING name, VALUE val)
+static int dump_uint64(VALUE val, LWES_BYTE_P buf, size_t *off)
 {
 	unsigned LONG_LONG tmp = NUM2ULL(val); /* can raise RangeError */
 	ID type = TYPE(val);
 
 	if ((type == T_FIXNUM && FIX2LONG(val) < 0) ||
-	    (type == T_BIGNUM && RTEST(rb_funcall(val, '<', 1, INT2FIX(0)))))
+	    (type == T_BIGNUM && RTEST(rb_funcall(val, '<', 1, INT2FIX(0))))) {
+		volatile VALUE raise_inspect;
+
 		rb_raise(rb_eRangeError, ":uint64 negative: %s",
-		         RSTRING_PTR(rb_inspect(val)));
+		         RAISE_INSPECT(val));
+	}
 
-	return lwes_event_set_U_INT_64(event, name, (LWES_U_INT_64)tmp);
+	lwesrb_dump_type(LWES_U_INT_64_TOKEN, buf, off);
+	return marshall_U_INT_64((LWES_U_INT_64)tmp, buf, MAX_MSG_SIZE, off);
 }
 
-static int set_int64(
-	struct lwes_event *event, LWES_CONST_SHORT_STRING name, VALUE val)
+static int dump_int64(VALUE val, LWES_BYTE_P buf, size_t *off)
 {
 	LONG_LONG tmp = NUM2LL(val); /* can raise RangeError */
 
-	return lwes_event_set_INT_64(event, name, (LWES_INT_64)tmp);
+	lwesrb_dump_type(LWES_INT_64_TOKEN, buf, off);
+	return marshall_INT_64((LWES_INT_64)tmp, buf, MAX_MSG_SIZE, off);
 }
 
-static int set_ip_addr(
-	struct lwes_event *event, LWES_CONST_SHORT_STRING name, VALUE val)
+static int dump_ip_addr(VALUE val, LWES_BYTE_P buf, size_t *off)
 {
+	LWES_IP_ADDR addr;
+	volatile VALUE raise_inspect;
+
 	switch (TYPE(val)) {
 	case T_STRING:
-	{
-		LWES_CONST_SHORT_STRING addr = RSTRING_PTR(val);
-		return lwes_event_set_IP_ADDR_w_string(event, name, addr);
-	}
+		addr.s_addr = inet_addr(RSTRING_PTR(val));
+		break;
 	case T_FIXNUM:
 	case T_BIGNUM:
-	{
-		LWES_IP_ADDR *addr = ALLOC(LWES_IP_ADDR); /* never fails */
-		int rv;
-
-		addr->s_addr = htonl(NUM2UINT(val));
-		rv = lwes_event_set_IP_ADDR(event, name, *addr);
-
-		if (rv < 0)
-			xfree(addr);
-		return rv;
-	}
+		addr.s_addr = htonl(NUM2UINT(val));
+		break;
 	default:
 		rb_raise(rb_eTypeError,
 		         ":ip_addr address must be String or Integer: %s",
-		         RSTRING_PTR(rb_inspect(val)));
+		         RAISE_INSPECT(val));
 	}
+	lwesrb_dump_type(LWES_IP_ADDR_TOKEN, buf, off);
+	return marshall_IP_ADDR(addr, buf, MAX_MSG_SIZE, off);
 }
 
 /* simple type => function dispatch map */
 static struct _type_fn_map {
 	ID type;
-	int (*fn)(struct lwes_event *, LWES_CONST_SHORT_STRING, VALUE);
+	int (*fn)(VALUE, LWES_BYTE_P, size_t *);
 } type_fn_map[] = {
-#define SYMFN(T) { (ID)&sym_##T, set_##T }
+#define SYMFN(T) { (ID)&sym_##T, dump_##T }
 	SYMFN(uint16),
 	SYMFN(int16),
 	SYMFN(uint32),
@@ -126,39 +129,47 @@ static struct _type_fn_map {
 #undef SYMFN
 };
 
-int lwesrb_event_set_num(
-	struct lwes_event *event,
-	LWES_CONST_SHORT_STRING name,
+/* used for Struct serialization where types are known ahead of time */
+static int dump_num(
 	LWES_TYPE type,
-	VALUE val)
+	VALUE val,
+	LWES_BYTE_P buf,
+	size_t *off)
 {
 	switch (type) {
-	case LWES_TYPE_U_INT_16: return set_uint16(event, name, val);
-	case LWES_TYPE_INT_16: return set_int16(event, name, val);
-	case LWES_TYPE_U_INT_32: return set_uint32(event, name, val);
-	case LWES_TYPE_INT_32: return set_int32(event, name, val);
-	case LWES_TYPE_U_INT_64: return set_uint64(event, name, val);
-	case LWES_TYPE_INT_64: return set_int64(event, name, val);
-	case LWES_TYPE_IP_ADDR: return set_ip_addr(event, name, val);
+	case LWES_TYPE_U_INT_16: return dump_uint16(val, buf, off);
+	case LWES_TYPE_INT_16: return dump_int16(val, buf, off);
+	case LWES_TYPE_U_INT_32: return dump_uint32(val, buf, off);
+	case LWES_TYPE_INT_32: return dump_int32(val, buf, off);
+	case LWES_TYPE_U_INT_64: return dump_uint64(val, buf, off);
+	case LWES_TYPE_INT_64: return dump_int64(val, buf, off);
+	case LWES_TYPE_IP_ADDR: return dump_ip_addr(val, buf, off);
 	default:
 		rb_raise(rb_eRuntimeError,
 			 "unknown LWES attribute type: 0x%02x", type);
 	}
-	assert("you should never get here (set_field)");
+	assert("you should never get here (dump_num)");
 	return -1;
 }
 
+void lwesrb_dump_num(LWES_BYTE type, VALUE val, LWES_BYTE_P buf, size_t *off)
+{
+	if (dump_num(type, val, buf, off) > 0)
+		return;
+	rb_raise(rb_eRuntimeError,
+	         "dumping numeric type 0x%02x, type failed", type);
+}
+
 /*
+ * used for Hash serialization
  * array contains two elements:
  *   [ symbolic_type, number ]
  * returns the return value of the underlying lwes_event_set_* call
  */
-int lwesrb_event_set_numeric(
-	struct lwes_event *event,
-	LWES_CONST_SHORT_STRING name,
-	VALUE array)
+void lwesrb_dump_num_ary(VALUE array, LWES_BYTE_P buf, size_t *off)
 {
-	int i;
+	volatile VALUE raise_inspect;
+	int i, rv;
 	struct _type_fn_map *head;
 	VALUE *ary;
 	ID type;
@@ -173,15 +184,19 @@ int lwesrb_event_set_numeric(
 
 	i = sizeof(type_fn_map) / sizeof(type_fn_map[0]);
 	for (head = type_fn_map; --i >= 0; head++) {
-		if (head->type == type)
-			return head->fn(event, name, ary[1]);
-	}
+		if (head->type != type)
+			continue;
 
-	rb_raise(rb_eArgError,
-	         "unknown type: %s",
-	         RSTRING_PTR(rb_inspect(type)));
+		rv = head->fn(ary[1], buf, off);
+		if (rv > 0)
+			return;
 
-	return -1;
+		rb_raise(rb_eRuntimeError,
+			 "dumping numeric type %s, type failed",
+			 RAISE_INSPECT(type));
+	}
+
+	rb_raise(rb_eArgError, "unknown type: %s", RAISE_INSPECT(type));
 }
 
 void lwesrb_init_numeric(void)

data/ext/lwes/type_db.c

@@ -103,11 +103,12 @@ struct lwes_event_type_db * lwesrb_get_type_db(VALUE self)
 	struct _tdb *tdb;
 
 	Data_Get_Struct(self, struct _tdb, tdb);
-	if (!tdb->db)
+	if (! tdb->db) {
+		volatile VALUE raise_inspect;
 		rb_raise(rb_eRuntimeError,
-		         "couldn't get lwes_type_db from %s",
-		         RSTRING_PTR(rb_inspect(self)));
-
+			 "couldn't get lwes_type_db from %s",
+			 RAISE_INSPECT(self));
+	}
 	return tdb->db;
 }
 

data/lib/lwes.rb

@@ -1,6 +1,6 @@
 module LWES
-  # version of our library, currently 0.3.1
-  VERSION = "0.3.1"
+  # version of our library, currently 0.4.0
+  VERSION = "0.4.0"
 
   autoload :TypeDB, "lwes/type_db"
   autoload :Struct, "lwes/struct"

data/lib/lwes/struct.rb

@@ -95,9 +95,21 @@ module LWES
       tmp.const_set :NAME, name.to_s
       ed = tmp.const_set :EVENT_DEF, {}
       event_def.each { |(field,type)| ed[field] = type }
-      type_list = event_def.map { |(field,type)| [ field, field.to_s, type ] }
+
+      # freeze since emitter.c can segfault if this ever changes
+      type_list = event_def.map do |(field,type)|
+        [ field, field.to_s.freeze, type ].freeze
+      end.freeze
       tmp.const_set :TYPE_LIST, type_list
 
+      aref_map = tmp.const_set :AREF_MAP, {}
+      type_list.each_with_index do |(field_sym,field_str,_),idx|
+        aref_map[field_sym] = aref_map[field_str] = idx
+      end
+
+      tmp.const_set :HAVE_ENCODING,
+                    type_list.include?([ :enc, 'enc', LWES::INT_16 ])
+
       defaults = options[:defaults] || {}
       defaults = tmp.const_set :DEFAULTS, defaults.dup
       tmp.class_eval(&block) if block_given?
@@ -122,6 +134,38 @@ class ::#{tmp.name}
   end
 end
 EOS
+
+    # avoid linear scans for large structs, not sure if 50 is a good enough
+    # threshold but it won't help for anything <= 10 since Ruby (or at least
+    # MRI) already optimizes those cases
+    if event_def.size > 50
+      eval <<EOS
+class ::#{tmp.name}
+  alias __aref []
+  alias __aset []=
+  def [](key)
+    __aref(key.kind_of?(Fixnum) ? key : AREF_MAP[key])
+  end
+
+  def []=(key, value)
+    __aset(key.kind_of?(Fixnum) ? key : AREF_MAP[key], value)
+  end
+end
+EOS
+      fast_methods = []
+      event_def.each_with_index do |(fld,type), idx|
+        next if idx <= 9
+        if idx != aref_map[fld]
+          raise LoadError, "event_def corrupt: #{event_def}"
+        end
+        fast_methods << "undef_method :#{fld}, :#{fld}=\n"
+        fast_methods << "\ndef #{fld}; __aref #{idx}; end\n"
+        fast_methods << "\ndef #{fld}=(val); __aset #{idx}, val ; end\n"
+      end
+
+      eval("class ::#{tmp.name}; #{fast_methods.join("\n")}\n end")
+    end
+
     tmp
     end
   end

data/lwes.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = %q{lwes}
-  s.version = "0.3.1"
+  s.version = "0.4.0"
   s.date = Time.now
   s.authors = ["Erik S. Chang", "Frank Maritato"]
   s.email = %q{lwes-devel@lists.sourceforge.net}

data/test/test_helper.rb

@@ -2,6 +2,7 @@ require 'pp'
 require 'tempfile'
 require 'test/unit'
 require 'lwes'
+GC.stress = true if ENV["GC_STRESS"].to_i != 0
 
 unless defined?(LISTENER_DEFAULTS)
   BEFORE_DELAY = ENV['BEFORE_DELAY'] ? ENV['BEFORE_DELAY'].to_f : 0.5

data/test/unit/test_emitter.rb

@@ -99,16 +99,6 @@ class TestEmitter < Test::Unit::TestCase
     assert_equal 1, lines.grep(/int_ip = 192.168.1.1/).size
   end
 
-  def TODO_emit_ip_addr_object
-    emitter = LWES::Emitter.new(@options)
-    event = { :ip => IPAddr.new("192.168.1.1") }
-    out = lwes_listener do
-      assert_nothing_raised { emitter.emit("IP", event) }
-    end
-    lines = out.readlines
-    assert_equal 1, lines.grep(/\bip = 192.168.1.1/).size
-  end
-
   def test_emit_junk
     emitter = LWES::Emitter.new(@options)
     assert_raises(ArgumentError) { emitter.emit("JUNK", :junk => %r{junk}) }
@@ -180,6 +170,48 @@ class TestEmitter < Test::Unit::TestCase
     }
   end
 
+  def test_emit_enc_sym
+    emitter = LWES::Emitter.new(@options)
+    tmp = {
+      :enc => 1,
+      :a => "b",
+      :c => "d",
+    }
+    out = lwes_listener do
+      assert_nothing_raised { emitter.emit("EncTest", tmp) }
+    end
+    lines = out.readlines
+    assert_equal 1, lines.grep(/\A\tenc = 1;\n\z/).size
+    assert_equal 1, lines.grep(/\A\ta = b;\n\z/).size
+    assert_equal 1, lines.grep(/\A\tc = d;\n\z/).size
+  end
+
+  def test_emit_enc_str
+    emitter = LWES::Emitter.new(@options)
+    tmp = {
+      :enc => 1,
+      :a => "b",
+      :c => "d",
+    }
+    out = lwes_listener do
+      assert_nothing_raised { emitter.emit("EncTest", tmp) }
+    end
+    lines = out.readlines
+    assert_equal 1, lines.grep(/\A\tenc = 1;\n\z/).size
+    assert_equal 1, lines.grep(/\A\ta = b;\n\z/).size
+    assert_equal 1, lines.grep(/\A\tc = d;\n\z/).size
+  end
+
+  def test_emit_huge_event
+    emitter = LWES::Emitter.new(@options)
+    tmp = {
+      :a => ("b" * (1024 * 1024)),
+    }
+    assert_raises(ArgumentError) do
+      emitter.emit("BigTest", tmp)
+    end
+  end
+
   def test_close
     emitter = LWES::Emitter.new(@options)
     assert_nil emitter.close

data/test/unit/test_struct.rb

@@ -57,6 +57,7 @@ class TestStruct < Test::Unit::TestCase
                          :class => :EVENT_BOOL,
                          :name => :EventBool)
     assert_equal "EVENT_BOOL", b.name
+    assert_equal false, b.const_get(:HAVE_ENCODING)
 
     e = assert_raises(RuntimeError) {
       LWES::Struct.new(:file=>MULTI_EVENT_ESF)
@@ -84,6 +85,7 @@ class TestStruct < Test::Unit::TestCase
     assert_equal "Event1", a.name
     assert a.const_get(:TYPE_DB).instance_of?(LWES::TypeDB)
     assert a.const_get(:EVENT_DEF).instance_of?(Hash)
+    assert a.const_get(:HAVE_ENCODING)
     assert_equal EXPECT_DB, a.const_get(:EVENT_DEF)
     assert_equal EXPECT_LIST, a.const_get(:TYPE_LIST)
     y = a.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: lwes
 version: !ruby/object:Gem::Version 
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors: 
 - Erik S. Chang
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-15 00:00:00 +00:00
+date: 2010-04-20 00:00:00 +00:00
 default_executable: 
 dependencies: []