wankel 0.1.0

data/ext/wankel/extconf.rb

@@ -0,0 +1,15 @@
+require 'mkmf'
+require 'rbconfig'
+
+$CFLAGS << '-Wall'
+
+if ARGV.include?('--coverage')
+  $CFLAGS << " -fprofile-arcs  -ftest-coverage"
+  $DLDFLAGS << ' --coverage'
+end
+
+if have_library('yajl')
+  create_makefile('wankel/wankel')
+else
+  puts "Couldn't find YAJL library"
+end

data/ext/wankel/wankel.c

@@ -0,0 +1,50 @@
+#include "wankel.h"
+
+static ID intern_new, intern_parse, intern_encode;
+
+static VALUE c_wankel, c_wankelParser, c_wankelEncoder, e_parseError, e_encodeError;
+
+// Class Methods =============================================================
+static VALUE wankel_parse(int argc, VALUE * argv, VALUE klass) {
+    VALUE parser, input, options, callback;
+    rb_scan_args(argc, argv, "11&", &input, &options, &callback);
+    
+    parser = rb_funcall(c_wankelParser, intern_new, 1, options);
+    return rb_funcall(parser, intern_parse, 2, input, callback);
+}
+
+static VALUE wankel_encode(int argc, VALUE * argv, VALUE klass) {
+    VALUE encoder, input, output, options;
+    rb_scan_args(argc, argv, "12", &input, &output, &options);
+	
+
+	if (TYPE(output) == T_HASH) {
+		encoder = rb_funcall(c_wankelEncoder, intern_new, 1, output);
+		return rb_funcall(encoder, intern_encode, 1, input);
+	} else {
+		encoder = rb_funcall(c_wankelEncoder, intern_new, 1, options);
+		return rb_funcall(encoder, intern_encode, 2, input, output);
+	}
+}
+
+void Init_wankel() {
+    c_wankel = rb_define_class("Wankel", rb_cObject);
+    e_parseError = rb_define_class_under(c_wankel, "ParseError", rb_eStandardError);
+    e_encodeError = rb_define_class_under(c_wankel, "EncodeError", rb_eStandardError);
+    
+    intern_new = rb_intern("new");
+    intern_parse = rb_intern("parse");
+	intern_encode = rb_intern("encode");
+
+    rb_define_singleton_method(c_wankel, "parse", wankel_parse, -1);
+    rb_define_singleton_method(c_wankel, "encode", wankel_encode, -1);
+	
+    rb_define_singleton_method(c_wankel, "load", wankel_parse, -1);
+    rb_define_singleton_method(c_wankel, "dump", wankel_encode, -1);
+    
+    c_wankelParser = Init_wankel_parser();
+    c_wankelEncoder = Init_wankel_encoder();
+    Init_wankel_sax_parser();
+    Init_wankel_sax_encoder();
+    Init_yajl_helpers();
+}

data/ext/wankel/wankel.h

@@ -0,0 +1,17 @@
+#ifndef WANKEL
+#define WANKEL
+
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include <yajl/yajl_common.h>
+#include <yajl/yajl_parse.h>
+
+#include "wankel_parser.h"
+#include "wankel_encoder.h"
+#include "wankel_sax_parser.h"
+#include "wankel_sax_encoder.h"
+#include "yajl_helpers.h"
+
+void Init_wankel();
+
+#endif

data/ext/wankel/wankel_encoder.c

@@ -0,0 +1,232 @@
+// TODO: i should use the Wankel::SaxEncoder?
+#include "wankel_encoder.h"
+
+static VALUE c_wankel, c_wankelEncoder, e_encodeError;
+
+static ID intern_to_s, intern_keys, intern_io_write, intern_to_json, intern_clone, intern_merge, intern_DEFAULTS;
+
+static ID sym_beautify, sym_indent_string, sym_validate_utf8, sym_escape_solidus;
+
+
+static void wankelEncoder_flush(yajl_gen g, VALUE io, int write_buffer_size);
+static void yajl_encode_part(yajl_gen g, VALUE obj, VALUE io, int write_buffer_size);
+static void wankelEncoder_flush(yajl_gen g, VALUE io, int write_buffer_size);
+
+/*
+ * Document-method: new
+ *
+ * call-seq: new([options])
+ *
+ * +:beautify+ generate indented (beautiful) output. Default `false`.
+ *
+ * +:indent_string+ Set an indent string which is used when yajl_gen_beautify
+ *                  is enabled. Maybe something like \\t or some number of
+ *                  spaces. The default is four spaces ' '.
+ *
+ * +:validate_utf8+ Normally the generator does not validate that strings you
+ *                  pass to it are valid UTF8. Enabling this option will cause
+ *                  it to do so.
+ *
+ * +:escape_solidus+ the forward solidus (slash or '/' in human) is not required
+ *                   to be escaped in json text. By default, YAJL will not escape
+ *                   it in the iterest of saving bytes. Setting this flag will
+ *                   cause YAJL to always escape '/' in generated JSON strings.
+ */
+static VALUE wankelEncoder_initialize(int argc, VALUE * argv, VALUE self) {
+    VALUE defaults = rb_const_get(c_wankel, intern_DEFAULTS);
+    VALUE options;
+    
+    rb_scan_args(argc, argv, "01", &options);
+    if(options == Qnil) {
+        rb_iv_set(self, "@options", rb_funcall(defaults, intern_clone, 0) );
+    } else {
+        Check_Type(options, T_HASH);
+        rb_iv_set(self, "@options", rb_funcall(defaults, intern_merge, 1, options) );
+    }
+
+    return self;
+}
+
+/*
+ * Document-method: encode
+ *
+ * call-seq: encode(obj[, io])
+ *
+ * +obj+ is the Ruby object to encode to JSON
+ *
+ * +io+ is an optional IO used to stream the encoded JSON string to. If no io
+ *      is specified the resulting JSON string is returned. If io is specified,
+ *      this method returns nil
+ */
+static VALUE wankelEncoder_encode(int argc, VALUE * argv, VALUE self) {
+	VALUE obj, io, options;
+	yajl_gen g;
+	yajl_alloc_funcs alloc_funcs;
+	yajl_gen_status status;
+	int write_buffer_size;
+    const unsigned char * buffer;
+    size_t len;
+
+	rb_scan_args(argc, argv, "11", &obj, &io);
+	options = rb_iv_get(self, "@options");
+	
+	alloc_funcs.malloc = yajl_helper_malloc;
+	alloc_funcs.realloc = yajl_helper_realloc;
+	alloc_funcs.free = yajl_helper_free;
+	g = yajl_gen_alloc(&alloc_funcs);
+	
+	yajl_gen_configure(g, options);
+	
+	if (io != Qnil && !rb_respond_to(io, intern_io_write)) {
+		rb_raise(e_encodeError, "output must be a an IO");
+	}
+	
+	write_buffer_size = FIX2INT(rb_hash_aref(options, ID2SYM(rb_intern("write_buffer_size"))));
+	
+	yajl_encode_part(g, obj, io, write_buffer_size);
+	
+	// TODO: add terminator here if desired
+	if (io == Qnil) {
+		status = yajl_gen_get_buf(g, &buffer, &len);
+		yajl_helper_check_gen_status(status);
+	    io = rb_str_new((const char *)buffer, len);
+		rb_enc_associate(io, rb_utf8_encoding());
+		yajl_gen_clear(g);
+		yajl_gen_free(g);
+		return io;
+	} else {
+		wankelEncoder_flush(g, io, 1);
+		yajl_gen_free(g);
+		return Qnil;
+	}	
+	return self;
+}
+
+void wankelEncoder_flush(yajl_gen g, VALUE io, int write_buffer_size) {
+	VALUE rbBuffer;
+    yajl_gen_status status;
+    const unsigned char * buffer;
+    size_t len;
+	
+	if (io != Qnil) {
+		status = yajl_gen_get_buf(g, &buffer, &len);
+		yajl_helper_check_gen_status(status);
+		
+		if (len >= (size_t)write_buffer_size) {
+			rbBuffer = rb_str_new((const char *)buffer, len);
+			rb_enc_associate(rbBuffer, rb_utf8_encoding());
+			rb_io_write(io, rbBuffer);
+			yajl_gen_clear(g);
+		}
+	}
+}
+
+void yajl_encode_part(yajl_gen g, VALUE obj, VALUE io, int write_buffer_size) {
+    size_t len;
+	int idx = 0;
+    VALUE keys, entry, str;
+    const char * cptr;
+	yajl_gen_status status;
+
+    switch (TYPE(obj)) {
+        case T_HASH:
+            status = yajl_gen_map_open(g);
+			yajl_helper_check_gen_status(status);
+			
+            keys = rb_funcall(obj, intern_keys, 0);
+            for(idx = 0; idx < RARRAY_LEN(keys); idx++) {
+               	entry = rb_ary_entry(keys, idx);
+                str = rb_funcall(entry, intern_to_s, 0); /* key must be a string */
+                /* the key */
+                yajl_encode_part(g, str, io, write_buffer_size);
+                /* the value */
+                yajl_encode_part(g, rb_hash_aref(obj, entry), io, write_buffer_size);
+            }
+
+            status = yajl_gen_map_close(g);
+			yajl_helper_check_gen_status(status);
+            break;
+        case T_ARRAY:
+            status = yajl_gen_array_open(g);
+			yajl_helper_check_gen_status(status);
+			
+            for(idx = 0; idx < RARRAY_LEN(obj); idx++) {
+                yajl_encode_part(g, rb_ary_entry(obj, idx), io, write_buffer_size);
+            }
+            status = yajl_gen_array_close(g);
+			yajl_helper_check_gen_status(status);
+            break;
+        case T_NIL:
+            status = yajl_gen_null(g);
+			yajl_helper_check_gen_status(status);
+            break;
+        case T_TRUE:
+            status = yajl_gen_bool(g, 1);
+			yajl_helper_check_gen_status(status);
+            break;
+        case T_FALSE:
+            status = yajl_gen_bool(g, 0);
+			yajl_helper_check_gen_status(status);
+            break;
+        case T_FIXNUM:
+        case T_FLOAT:
+        case T_BIGNUM:
+            str = rb_funcall(obj, intern_to_s, 0);
+            cptr = RSTRING_PTR(str);
+            len = RSTRING_LEN(str);
+            if (memcmp(cptr, "NaN", 3) == 0 || memcmp(cptr, "Infinity", 8) == 0 || memcmp(cptr, "-Infinity", 9) == 0) {
+                rb_raise(e_encodeError, "'%s' is an invalid number", cptr);
+            }
+            status = yajl_gen_number(g, cptr, len);
+			yajl_helper_check_gen_status(status);
+            break;
+        case T_STRING:
+            cptr = RSTRING_PTR(obj);
+            len = RSTRING_LEN(obj);
+            status = yajl_gen_string(g, (const unsigned char *)cptr, len);
+			yajl_helper_check_gen_status(status);
+            break;
+        default:
+            if (rb_respond_to(obj, intern_to_json)) {
+                str = rb_funcall(obj, intern_to_json, 0);
+                Check_Type(str, T_STRING);
+                cptr = RSTRING_PTR(str);
+                len = RSTRING_LEN(str);
+                status = yajl_gen_number(g, cptr, len);
+				yajl_helper_check_gen_status(status);
+            } else {
+                str = rb_funcall(obj, intern_to_s, 0);
+                Check_Type(str, T_STRING);
+                cptr = RSTRING_PTR(str);
+                len = RSTRING_LEN(str);
+                status = yajl_gen_string(g, (const unsigned char *)cptr, len);
+				yajl_helper_check_gen_status(status);
+            }
+            break;
+    }
+	
+	wankelEncoder_flush(g, io, write_buffer_size);
+}
+
+ID Init_wankel_encoder() {
+    c_wankel = rb_const_get(rb_cObject, rb_intern("Wankel"));
+    c_wankelEncoder = rb_define_class_under(c_wankel, "Encoder", rb_cObject);
+    e_encodeError = rb_const_get(c_wankel, rb_intern("EncodeError"));
+
+    rb_define_method(c_wankelEncoder, "initialize", wankelEncoder_initialize, -1);
+    rb_define_method(c_wankelEncoder, "encode", wankelEncoder_encode, -1);
+    
+	intern_to_s = rb_intern("to_s");
+	intern_io_write = rb_intern("write");
+	intern_to_json = rb_intern("to_json");
+	intern_keys = rb_intern("keys");
+    intern_clone = rb_intern("clone");
+    intern_merge = rb_intern("merge");
+    intern_DEFAULTS = rb_intern("DEFAULTS");
+    sym_beautify = ID2SYM(rb_intern("beautify"));
+    sym_indent_string = ID2SYM(rb_intern("indent_string"));
+    sym_validate_utf8 = ID2SYM(rb_intern("validate_utf8"));
+    sym_escape_solidus = ID2SYM(rb_intern("escape_solidus"));
+    
+    return c_wankelEncoder;
+}

data/ext/wankel/wankel_encoder.h

@@ -0,0 +1,13 @@
+#ifndef WANKEL_ENCODER
+#define WANKEL_ENCODER
+
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include <yajl/yajl_common.h>
+#include <yajl/yajl_gen.h>
+
+#include "yajl_helpers.h"
+
+ID Init_wankel_encoder();
+
+#endif

data/ext/wankel/wankel_parser.c

@@ -0,0 +1,345 @@
+#include "wankel_parser.h"
+
+// Callbacks =================================================================
+static int wankel_parse_callback_on_null(void *ctx);
+static int wankel_parse_callback_on_boolean(void *ctx, int boolVal);
+// static int wankel_parse_callback_on_integer(void *ctx, long long integerVal);
+// static int wankel_parse_callback_on_double(void *ctx, double doubleVal);
+static int wankel_parse_callback_on_number(void *ctx, const char * numberVal, size_t numberLen);
+static int wankel_parse_callback_on_string(void *ctx, const unsigned char * stringVal, size_t stringLen);
+static int wankel_parse_callback_on_map_start(void *ctx);
+static int wankel_parse_callback_on_map_key(void *ctx, const unsigned char * key, size_t keyLen);
+static int wankel_parse_callback_on_map_end(void *ctx);
+static int wankel_parse_callback_on_array_start(void *ctx);
+static int wankel_parse_callback_on_array_end(void *ctx);
+
+static yajl_callbacks callbacks = {
+    wankel_parse_callback_on_null,
+    wankel_parse_callback_on_boolean,
+    NULL,
+    NULL,
+    wankel_parse_callback_on_number,
+    wankel_parse_callback_on_string,
+    wankel_parse_callback_on_map_start,
+    wankel_parse_callback_on_map_key,
+    wankel_parse_callback_on_map_end,
+    wankel_parse_callback_on_array_start,
+    wankel_parse_callback_on_array_end
+};
+
+// Ruby GC ===================================================================
+static VALUE wankel_alloc(VALUE klass);
+static void wankel_free(void * handle);
+static void wankel_mark(void * handle);
+
+static ID intern_io_read, intern_clone, intern_merge, intern_call,
+          intern_DEFAULTS, sym_multiple_values;
+   
+static ID sym_read_buffer_size, sym_symbolize_keys; 
+
+static VALUE c_wankel, c_wankelParser, e_parseError, e_encodeError;
+
+/*
+ * Document-method: new
+ *
+ * call-seq: new([options])
+ *
+ * +:symbolize_keys+ will turn hash keys into Ruby symbols, defaults to false.
+ *                 Default `false`.
+ *
+ * +:allow_comments+ will ignore javascript style comments in JSON input.
+ *                 Default `false`.
+ *
+ * +:validate_strings+ will verify that all strings in JSON input are valid UTF8
+ *                   and will emit a parse error if this is not so. This option
+ * 					 makes parsing slightly more expensive (~7% depending on
+ *                   processor and compiler in use). Default `false`.
+ *
+ * +:allow_trailing_garbage+ will ensure the entire input text was consumed and
+ *                         will raise an error otherwise. Default `false`.
+ *
+ * +:multiple_values+ allow multiple values to be parsed by a single parser. The
+ *                  entire text must be valid JSON, and values can be seperated
+ *                  by any kind of whitespace. Default `false`.
+ *
+ * +:allow_partial_values+ check that the top level value was completely consumed/
+ *                       Default `false`.
+ *
+ * 
+ * +:read_buffer_size+ is the size of chunk that will be parsed off the input
+ *                     (if it's an IO) for each loop of the parsing process.
+ *                     8092 is a good balance between the different types of
+ *                     streams (off disk, off a socket, etc...), but this option
+ *                     is here so the caller can better tune their parsing depending
+ *                     on the type of stream being passed. A larger read buffer
+ *                     will perform better for files off disk, where as a smaller
+ *                     size may be more efficient for reading off of a socket
+ *                     directly.
+ */
+static VALUE wankelParser_initialize(int argc, VALUE * argv, VALUE self) {
+    VALUE defaults = rb_const_get(c_wankel, intern_DEFAULTS);
+    VALUE options, rbufsize;
+    wankel_parser * p;
+    
+    rb_scan_args(argc, argv, "01", &options);
+    if(options == Qnil) {
+        rb_iv_set(self, "@options", rb_funcall(defaults, intern_clone, 0) );
+    } else {
+        Check_Type(options, T_HASH);
+        rb_iv_set(self, "@options", rb_funcall(defaults, intern_merge, 1, options) );
+    }
+    options = rb_iv_get(self, "@options");
+    rbufsize = rb_hash_aref(options, sym_read_buffer_size);
+    Check_Type(rbufsize, T_FIXNUM);
+	
+    Data_Get_Struct(self, wankel_parser, p);
+    p->alloc_funcs.malloc = yajl_helper_malloc;
+    p->alloc_funcs.realloc = yajl_helper_realloc;
+    p->alloc_funcs.free = yajl_helper_free;
+    p->rbufsize = rbufsize;
+
+    if(rb_hash_aref(options, sym_symbolize_keys) == Qtrue) {
+        p->symbolize_keys = 1;
+    } else {
+        p->symbolize_keys = 0;
+    }
+    
+    return self;
+}
+
+/*
+ * Document-method: parse
+ *
+ * call-seq: parse(input[, &block])
+ *
+ * input can either be a String or and IO Object
+ *
+ * If a block is passed, it is called when the input is finished parsing. If
+ * parsing multiple json values in an input it is called once for each value
+ *
+ */
+static VALUE wankelParser_parse(int argc, VALUE * argv, VALUE self) {
+    const char * cptr;
+    unsigned int len;
+    yajl_status status;
+    wankel_parser * p;
+    VALUE input, callback;
+    VALUE options = rb_iv_get(self, "@options");
+    rb_scan_args(argc, argv, "11", &input, &callback); // Hack, cuz i'm not sure how to call a method with a block from c
+    
+    if(callback == Qnil && rb_block_given_p()) {
+        callback = rb_block_proc();
+    }
+        
+    Data_Get_Struct(self, wankel_parser, p);
+    p->h = yajl_alloc(&callbacks, &p->alloc_funcs, (void *)p);
+    yajl_configure(p->h, options);
+    p->callback = callback;
+    p->stack = rb_ary_new();
+    p->stack_index = 0;
+    if (TYPE(input) == T_STRING) {
+        cptr = RSTRING_PTR(input);
+        len = (unsigned int)RSTRING_LEN(input);
+        status = yajl_parse(p->h, (const unsigned char*)cptr, len);
+        yajl_helper_check_status(p->h, status, 1, (const unsigned char*)cptr, len);
+    } else if (rb_respond_to(input, intern_io_read)) {
+        VALUE chunk = rb_str_new(0, NUM2LONG(p->rbufsize));
+        while (rb_funcall(input, intern_io_read, 2, p->rbufsize, chunk) != Qnil) {
+            cptr = RSTRING_PTR(chunk);
+            len = (unsigned int)RSTRING_LEN(chunk);
+            status = yajl_parse(p->h, (const unsigned char*)cptr, len);
+            yajl_helper_check_status(p->h, status, 1, (const unsigned char*)cptr, len);
+        }
+    } else {
+        rb_raise(e_parseError, "input must be a string or an IO");
+    }
+    
+    status = yajl_complete_parse(p->h);
+    if(status != yajl_status_ok) {
+        rb_raise(e_parseError, "Error completing parse");
+    }
+    
+    if(rb_block_given_p()) {
+        return Qnil;
+    } else if(rb_hash_aref(options, sym_multiple_values) == Qtrue) {
+        return p->stack;
+    } else {
+        return rb_ary_pop(p->stack);
+    }
+}
+
+ID Init_wankel_parser() {
+    c_wankel = rb_const_get(rb_cObject, rb_intern("Wankel"));
+    c_wankelParser = rb_define_class_under(c_wankel, "Parser", rb_cObject);
+    e_parseError = rb_const_get(c_wankel, rb_intern("ParseError"));
+    e_encodeError = rb_const_get(c_wankel, rb_intern("EncodeError"));
+
+    rb_define_alloc_func(c_wankelParser, wankel_alloc);
+    rb_define_method(c_wankelParser, "initialize", wankelParser_initialize, -1);
+    rb_define_method(c_wankelParser, "parse", wankelParser_parse, -1);
+
+    intern_io_read = rb_intern("read");
+    intern_clone = rb_intern("clone");
+    intern_merge = rb_intern("merge");
+    intern_call = rb_intern("call");
+    intern_DEFAULTS = rb_intern("DEFAULTS");
+    sym_read_buffer_size = ID2SYM(rb_intern("read_buffer_size"));
+    sym_symbolize_keys = ID2SYM(rb_intern("symbolize_keys"));
+    sym_multiple_values = ID2SYM(rb_intern("multiple_values"));
+    
+    return c_wankelParser;
+}
+// Ruby GC ===================================================================
+static VALUE wankel_alloc(VALUE klass) {
+	VALUE self;
+    wankel_parser * p;
+	self = Data_Make_Struct(klass, wankel_parser, wankel_mark, wankel_free, p);
+	p->h = 0;
+	return self;
+}
+
+static void wankel_free(void * handle) {
+    wankel_parser * p = handle;
+	if (p->h){
+		yajl_free(p->h);
+	}
+}
+
+static void wankel_mark(void * handle) {
+    wankel_parser * p = handle;
+    rb_gc_mark(p->stack);
+    rb_gc_mark(p->rbufsize);
+}
+
+// Parse Builder =============================================================
+static void wankel_builder_push(void *ctx, VALUE val) {
+    int len;
+    wankel_parser * p = ctx;
+    VALUE lastEntry, hash;
+    
+    if (p->stack_index > 0) {
+        len = (int)RARRAY_LEN(p->stack);
+        lastEntry = rb_ary_entry(p->stack, len-1);
+        switch (TYPE(lastEntry)) {
+            case T_ARRAY:
+                rb_ary_push(lastEntry, val);
+                if (TYPE(val) == T_HASH || TYPE(val) == T_ARRAY) {
+                    rb_ary_push(p->stack, val);
+                    p->stack_index++;
+                }
+                break;
+            case T_HASH:
+                rb_hash_aset(lastEntry, val, Qnil);
+                rb_ary_push(p->stack, val);
+                p->stack_index++;
+                break;
+            case T_STRING:
+            case T_SYMBOL:
+                hash = rb_ary_entry(p->stack, len-2);
+                if (TYPE(hash) == T_HASH) {
+                    rb_hash_aset(hash, lastEntry, val);
+                    rb_ary_pop(p->stack);
+                    p->stack_index--;
+                    if (TYPE(val) == T_HASH || TYPE(val) == T_ARRAY) {
+                        rb_ary_push(p->stack, val);
+                        p->stack_index++;
+                    }
+                }
+                break;
+        }
+    } else {
+        rb_ary_push(p->stack, val);
+        p->stack_index++;
+    }
+}
+
+static int wankel_parse_callback_on_null(void *ctx) {
+	wankel_builder_push(ctx, Qnil);
+    return 1;
+}
+
+static int wankel_parse_callback_on_boolean(void *ctx, int boolVal) {
+	wankel_builder_push(ctx, boolVal ? Qtrue : Qfalse);
+    return 1;
+}
+
+// static int wankel_parse_callback_on_integer(void *ctx, long long integerVal) {
+// 	wankel_builder_push(ctx, LL2NUM(integerVal));
+//     return 1;
+// }
+// static int wankel_parse_callback_on_double(void *ctx, double doubleVal) {
+// 	wankel_builder_push(ctx, rb_float_new(doubleVal));
+//     return 1;
+// }
+static int wankel_parse_callback_on_number(void * ctx, const char * numberVal, size_t numberLen){
+	char buf[numberLen+1];
+    buf[numberLen] = 0;
+    memcpy(buf, numberVal, numberLen);
+
+	if (memchr(buf, '.', numberLen) || memchr(buf, 'e', numberLen) || memchr(buf, 'E', numberLen)) {
+		wankel_builder_push(ctx, rb_float_new(strtod(buf, NULL)));
+	} else {
+		wankel_builder_push(ctx, rb_cstr2inum(buf, 10));
+	}
+	return 1;
+}
+static int wankel_parse_callback_on_string(void *ctx, const unsigned char * stringVal, size_t stringLen) {
+	VALUE str = rb_str_new((const char *)stringVal, stringLen);
+	rb_encoding *default_internal_enc = rb_default_internal_encoding();
+    rb_enc_associate(str, rb_utf8_encoding());
+    if (default_internal_enc) {
+      str = rb_str_export_to_enc(str, default_internal_enc);
+    }
+
+	wankel_builder_push(ctx, str);
+    return 1;
+}
+static int wankel_parse_callback_on_map_start(void *ctx) {
+	wankel_builder_push(ctx, rb_hash_new());
+    return 1;
+}
+static int wankel_parse_callback_on_map_key(void *ctx, const unsigned char * key, size_t keyLen) {
+	wankel_parser * p = ctx;
+	rb_encoding *default_internal_enc = rb_default_internal_encoding();
+	VALUE str = rb_str_new((const char *)key, keyLen);
+	rb_enc_associate(str, rb_utf8_encoding());
+    if (default_internal_enc) {
+      str = rb_str_export_to_enc(str, default_internal_enc);
+    }
+
+	if(p->symbolize_keys) {
+		wankel_builder_push(ctx, ID2SYM(rb_to_id(str)) );
+	} else {
+		wankel_builder_push(ctx, str);
+	}
+	
+    return 1;
+}
+static int wankel_parse_callback_on_map_end(void *ctx) {
+    wankel_parser * p = ctx;
+    p->stack_index--;
+    
+    if(p->stack_index > 0) {
+        rb_ary_pop(p->stack);
+    } else if(p->stack_index == 0 && p->callback != Qnil) {
+        rb_funcall(p->callback, intern_call, 1, rb_ary_pop(p->stack));
+    }
+    
+    return 1;
+}
+static int wankel_parse_callback_on_array_start(void *ctx) {
+	wankel_builder_push(ctx, rb_ary_new());
+    return 1;
+}
+static int wankel_parse_callback_on_array_end(void *ctx) {
+	wankel_parser * p = ctx;
+    p->stack_index--;
+    
+    if(p->stack_index > 0) {
+        rb_ary_pop(p->stack);
+    } else if(p->stack_index == 0 && p->callback != Qnil) {
+        rb_funcall(p->callback, intern_call, 1, rb_ary_pop(p->stack));
+    }
+
+    return 1;
+}

data/ext/wankel/wankel_parser.h

@@ -0,0 +1,26 @@
+#ifndef WANKEL_PARSER
+#define WANKEL_PARSER
+
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include <yajl/yajl_common.h>
+#include <yajl/yajl_parse.h>
+
+#include "yajl_helpers.h"
+
+ID Init_wankel_parser();
+
+typedef struct {
+    yajl_handle h;
+    yajl_alloc_funcs alloc_funcs;
+	VALUE stack;
+	int stack_index;
+	int symbolize_keys;
+	VALUE rbufsize;
+	VALUE callback;
+	VALUE last_entry;
+} wankel_parser;
+    
+
+
+#endif