brianmario-yajl-ruby 0.4.6 → 0.4.7

data/ext/yajl_buf.h

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2007-2009, Lloyd Hilaiel.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ * 
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ * 
+ *  3. Neither the name of Lloyd Hilaiel nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */ 
+
+#ifndef __YAJL_BUF_H__
+#define __YAJL_BUF_H__
+
+#include "api/yajl_common.h"
+#include "yajl_alloc.h"
+
+/*
+ * Implementation/performance notes.  If this were moved to a header
+ * only implementation using #define's where possible we might be 
+ * able to sqeeze a little performance out of the guy by killing function
+ * call overhead.  YMMV.
+ */
+
+/**
+ * yajl_buf is a buffer with exponential growth.  the buffer ensures that
+ * you are always null padded.
+ */
+typedef struct yajl_buf_t * yajl_buf;
+
+/* allocate a new buffer */
+yajl_buf yajl_buf_alloc(yajl_alloc_funcs * alloc);
+
+/* free the buffer */
+void yajl_buf_free(yajl_buf buf);
+
+/* append a number of bytes to the buffer */
+void yajl_buf_append(yajl_buf buf, const void * data, unsigned int len);
+
+/* empty the buffer */
+void yajl_buf_clear(yajl_buf buf);
+
+/* get a pointer to the beginning of the buffer */
+const unsigned char * yajl_buf_data(yajl_buf buf);
+
+/* get the length of the buffer */
+unsigned int yajl_buf_len(yajl_buf buf);
+
+/* truncate the buffer */
+void yajl_buf_truncate(yajl_buf buf, unsigned int len);
+
+#endif

data/ext/yajl_bytestack.h

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2009, Lloyd Hilaiel.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ * 
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ * 
+ *  3. Neither the name of Lloyd Hilaiel nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */ 
+
+/*
+ * A header only implementation of a simple stack of bytes, used in YAJL
+ * to maintain parse state.
+ */
+
+#ifndef __YAJL_BYTESTACK_H__
+#define __YAJL_BYTESTACK_H__
+
+#include "api/yajl_common.h"
+
+#define YAJL_BS_INC 128
+
+typedef struct yajl_bytestack_t
+{
+    unsigned char * stack;
+    unsigned int size;
+    unsigned int used;
+    yajl_alloc_funcs * yaf;
+} yajl_bytestack;
+
+/* initialize a bytestack */
+#define yajl_bs_init(obs, _yaf) {               \
+        (obs).stack = NULL;                     \
+        (obs).size = 0;                         \
+        (obs).used = 0;                         \
+        (obs).yaf = (_yaf);                     \
+    }                                           \
+
+
+/* initialize a bytestack */
+#define yajl_bs_free(obs)                 \
+    if ((obs).stack) (obs).yaf->free((obs).yaf->ctx, (obs).stack);   
+
+#define yajl_bs_current(obs)               \
+    (assert((obs).used > 0), (obs).stack[(obs).used - 1])
+
+#define yajl_bs_push(obs, byte) {                       \
+    if (((obs).size - (obs).used) == 0) {               \
+        (obs).size += YAJL_BS_INC;                      \
+        (obs).stack = (obs).yaf->realloc((obs).yaf->ctx,\
+                                         (void *) (obs).stack, (obs).size);\
+    }                                                   \
+    (obs).stack[((obs).used)++] = (byte);               \
+}
+    
+/* removes the top item of the stack, returns nothing */
+#define yajl_bs_pop(obs) { ((obs).used)--; }
+
+#define yajl_bs_set(obs, byte)                          \
+    (obs).stack[((obs).used) - 1] = (byte);             
+    
+
+#endif

data/ext/yajl_encode.c

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2007-2009, Lloyd Hilaiel.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ * 
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ * 
+ *  3. Neither the name of Lloyd Hilaiel nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */ 
+
+#include "yajl_encode.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+static void CharToHex(unsigned char c, char * hexBuf)
+{
+    const char * hexchar = "0123456789ABCDEF";
+    hexBuf[0] = hexchar[c >> 4];
+    hexBuf[1] = hexchar[c & 0x0F];
+}
+
+void
+yajl_string_encode(yajl_buf buf, const unsigned char * str,
+                   unsigned int len)
+{
+    unsigned int beg = 0;
+    unsigned int end = 0;    
+    char hexBuf[7];
+    hexBuf[0] = '\\'; hexBuf[1] = 'u'; hexBuf[2] = '0'; hexBuf[3] = '0';
+    hexBuf[6] = 0;
+
+    while (end < len) {
+        const char * escaped = NULL;
+        switch (str[end]) {
+            case '\r': escaped = "\\r"; break;
+            case '\n': escaped = "\\n"; break;
+            case '\\': escaped = "\\\\"; break;
+            /* case '/': escaped = "\\/"; break; */
+            case '"': escaped = "\\\""; break;
+            case '\f': escaped = "\\f"; break;
+            case '\b': escaped = "\\b"; break;
+            case '\t': escaped = "\\t"; break;
+            default:
+                if ((unsigned char) str[end] < 32) {
+                    CharToHex(str[end], hexBuf + 4);
+                    escaped = hexBuf;
+                }
+                break;
+        }
+        if (escaped != NULL) {
+            yajl_buf_append(buf, str + beg, end - beg);
+            yajl_buf_append(buf, escaped, strlen(escaped));
+            beg = ++end;
+        } else {
+            ++end;
+        }
+    }
+    yajl_buf_append(buf, str + beg, end - beg);
+}
+
+static void hexToDigit(unsigned int * val, const unsigned char * hex)
+{
+    unsigned int i;
+    for (i=0;i<4;i++) {
+        unsigned char c = hex[i];
+        if (c >= 'A') c = (c & ~0x20) - 7;
+        c -= '0';
+        assert(!(c & 0xF0));
+        *val = (*val << 4) | c;
+    }
+}
+
+static void Utf32toUtf8(unsigned int codepoint, char * utf8Buf) 
+{
+    if (codepoint < 0x80) {
+        utf8Buf[0] = (char) codepoint;
+        utf8Buf[1] = 0;
+    } else if (codepoint < 0x0800) {
+        utf8Buf[0] = (char) ((codepoint >> 6) | 0xC0);
+        utf8Buf[1] = (char) ((codepoint & 0x3F) | 0x80);
+        utf8Buf[2] = 0;
+    } else if (codepoint < 0x10000) {
+        utf8Buf[0] = (char) ((codepoint >> 12) | 0xE0);
+        utf8Buf[1] = (char) (((codepoint >> 6) & 0x3F) | 0x80);
+        utf8Buf[2] = (char) ((codepoint & 0x3F) | 0x80);
+        utf8Buf[3] = 0;
+    } else if (codepoint < 0x200000) {
+        utf8Buf[0] =(char)((codepoint >> 18) | 0xF0);
+        utf8Buf[1] =(char)(((codepoint >> 12) & 0x3F) | 0x80);
+        utf8Buf[2] =(char)(((codepoint >> 6) & 0x3F) | 0x80);
+        utf8Buf[3] =(char)((codepoint & 0x3F) | 0x80);
+        utf8Buf[4] = 0;
+    } else {
+        utf8Buf[0] = '?';
+        utf8Buf[1] = 0;
+    }
+}
+
+void yajl_string_decode(yajl_buf buf, const unsigned char * str,
+                        unsigned int len)
+{
+    unsigned int beg = 0;
+    unsigned int end = 0;    
+
+    while (end < len) {
+        if (str[end] == '\\') {
+            char utf8Buf[5];
+            const char * unescaped = "?";
+            yajl_buf_append(buf, str + beg, end - beg);
+            switch (str[++end]) {
+                case 'r': unescaped = "\r"; break;
+                case 'n': unescaped = "\n"; break;
+                case '\\': unescaped = "\\"; break;
+                case '/': unescaped = "/"; break;
+                case '"': unescaped = "\""; break;
+                case 'f': unescaped = "\f"; break;
+                case 'b': unescaped = "\b"; break;
+                case 't': unescaped = "\t"; break;
+                case 'u': {
+                    unsigned int codepoint = 0;
+                    hexToDigit(&codepoint, str + ++end);
+                    end+=3;
+                    /* check if this is a surrogate */
+                    if ((codepoint & 0xFC00) == 0xD800) {
+                        end++;
+                        if (str[end] == '\\' && str[end + 1] == 'u') {
+                            unsigned int surrogate = 0;
+                            hexToDigit(&surrogate, str + end + 2);
+                            codepoint =
+                                (((codepoint & 0x3F) << 10) | 
+                                 ((((codepoint >> 6) & 0xF) + 1) << 16) | 
+                                 (surrogate & 0x3FF));
+                            end += 5;
+                        } else {
+                            unescaped = "?";
+                            break;
+                        }
+                    }
+                    
+                    Utf32toUtf8(codepoint, utf8Buf);
+                    unescaped = utf8Buf;
+                    break;
+                }
+                default:
+                    assert("this should never happen" == NULL);
+            }
+            yajl_buf_append(buf, unescaped, strlen(unescaped));
+            beg = ++end;
+        } else {
+            end++;
+        }
+    }
+    yajl_buf_append(buf, str + beg, end - beg);
+}

data/ext/yajl_encode.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2007-2009, Lloyd Hilaiel.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ * 
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ * 
+ *  3. Neither the name of Lloyd Hilaiel nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */ 
+
+#ifndef __YAJL_ENCODE_H__
+#define __YAJL_ENCODE_H__
+
+#include "yajl_buf.h"
+
+void yajl_string_encode(yajl_buf buf, const unsigned char * str,
+                        unsigned int length);
+
+void yajl_string_decode(yajl_buf buf, const unsigned char * str,
+                        unsigned int length);
+
+#endif

data/ext/yajl_ext.c

@@ -0,0 +1,283 @@
+#include "yajl_ext.h"
+
+void check_and_fire_callback(void * ctx) {
+    yajl_status stat;
+    
+    if (RARRAY_LEN((VALUE)ctx) == 1 && parse_complete_callback != Qnil) {
+        // parse any remaining buffered data
+        stat = yajl_parse_complete(chunkedParser);
+        
+        rb_funcall(parse_complete_callback, intern_call, 1, rb_ary_pop((VALUE)ctx));
+    }
+}
+
+void set_static_value(void * ctx, VALUE val) {
+    VALUE len = RARRAY_LEN((VALUE)ctx);
+    
+    if (len > 0) {
+        VALUE lastEntry = rb_ary_entry((VALUE)ctx, len-1);
+        VALUE hash;
+        switch (TYPE(lastEntry)) {
+            case T_ARRAY:
+                rb_ary_push(lastEntry, val);
+                if (TYPE(val) == T_HASH || TYPE(val) == T_ARRAY) {
+                    rb_ary_push((VALUE)ctx, val);
+                }
+                break;
+            case T_HASH:
+                rb_hash_aset(lastEntry, val, Qnil);
+                rb_ary_push((VALUE)ctx, val);
+                break;
+            case T_STRING:
+                hash = rb_ary_entry((VALUE)ctx, len-2);
+                if (TYPE(hash) == T_HASH) {
+                    rb_hash_aset(hash, lastEntry, val);
+                    rb_ary_pop((VALUE)ctx);
+                    if (TYPE(val) == T_HASH || TYPE(val) == T_ARRAY) {
+                        rb_ary_push((VALUE)ctx, val);
+                    }
+                }
+                break;
+        }
+    } else {
+        rb_ary_push((VALUE)ctx, val);
+    }
+}
+
+void encode_part(yajl_gen hand, VALUE obj, VALUE io) {
+    VALUE str, outBuff, otherObj;
+    int objLen;
+    int idx = 0;
+    const unsigned char * buffer;
+    unsigned int len;
+    yajl_gen_get_buf(hand, &buffer, &len);
+    outBuff = rb_str_new((const char *)buffer, len);
+    rb_io_write(io, outBuff);
+    yajl_gen_clear(hand);
+    
+    switch (TYPE(obj)) {
+        case T_HASH:
+            yajl_gen_map_open(hand);
+            
+            // TODO: itterate through keys in the hash
+            VALUE keys = rb_funcall(obj, intern_keys, 0);
+            VALUE entry;
+            for(idx=0; idx<RARRAY_LEN(keys); idx++) {
+                entry = rb_ary_entry(keys, idx);
+                // the key
+                encode_part(hand, entry, io);
+                // the value
+                encode_part(hand, rb_hash_aref(obj, entry), io);
+            }
+            
+            yajl_gen_map_close(hand);
+            break;
+        case T_ARRAY:
+            yajl_gen_array_open(hand);
+            for(idx=0; idx<RARRAY_LEN(obj); idx++) {
+                otherObj = rb_ary_entry(obj, idx);
+                encode_part(hand, otherObj, io);
+            }
+            yajl_gen_array_close(hand);
+            break;
+        case T_NIL:
+            yajl_gen_null(hand);
+            break;
+        case T_TRUE:
+            yajl_gen_bool(hand, 1);
+            break;
+        case T_FALSE:
+            yajl_gen_bool(hand, 0);
+            break;
+        case T_FIXNUM:
+        case T_FLOAT:
+        case T_BIGNUM:
+            str = rb_funcall(obj, intern_to_s, 0);
+            objLen = RSTRING_LEN(str);
+            yajl_gen_number(hand, RSTRING_PTR(str), (unsigned int)objLen);
+            break;
+        default:
+            str = rb_funcall(obj, intern_to_s, 0);
+            objLen = RSTRING_LEN(str);
+            yajl_gen_string(hand, (const unsigned char *)RSTRING_PTR(str), (unsigned int)objLen);
+            break;
+    }
+}
+
+static int found_null(void * ctx) {
+    set_static_value(ctx, Qnil);
+    check_and_fire_callback(ctx);
+    return 1;
+}
+
+static int found_boolean(void * ctx, int boolean) {
+    set_static_value(ctx, boolean ? Qtrue : Qfalse);
+    check_and_fire_callback(ctx);
+    return 1;
+}
+
+static int found_number(void * ctx, const char * numberVal, unsigned int numberLen) {
+    VALUE subString = rb_str_new(numberVal, numberLen);
+    if (strstr(RSTRING_PTR(subString), ".") != NULL || strstr(RSTRING_PTR(subString), "e") != NULL || strstr(RSTRING_PTR(subString), "E") != NULL) {
+        set_static_value(ctx, rb_Float(subString));
+    } else {
+        set_static_value(ctx, rb_Integer(subString));
+    }
+    check_and_fire_callback(ctx);
+    return 1;
+}
+
+static int found_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
+    set_static_value(ctx, rb_str_new((const char *)stringVal, stringLen));
+    check_and_fire_callback(ctx);
+    return 1;
+}
+
+static int found_hash_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
+    set_static_value(ctx, rb_str_new((const char *)stringVal, stringLen));
+    return 1;
+}
+
+static int found_start_hash(void * ctx) {
+    set_static_value(ctx, rb_hash_new());
+    return 1;
+}
+
+static int found_end_hash(void * ctx) {
+    if (RARRAY_LEN((VALUE)ctx) > 1) {
+        rb_ary_pop((VALUE)ctx);
+    }
+    check_and_fire_callback(ctx);
+    return 1;
+}
+
+static int found_start_array(void * ctx) {
+    set_static_value(ctx, rb_ary_new());
+    return 1;
+}
+
+static int found_end_array(void * ctx) {
+    if (RARRAY_LEN((VALUE)ctx) > 1) {
+        rb_ary_pop((VALUE)ctx);
+    }
+    check_and_fire_callback(ctx);
+    return 1;
+}
+
+static VALUE t_setParseComplete(VALUE self, VALUE callback) {
+    parse_complete_callback = callback;
+    return Qnil;
+}
+
+static VALUE t_parseSome(VALUE self, VALUE string) {
+    yajl_status stat;
+    
+    if (string == Qnil) {
+        rb_raise(cParseError, "%s", "Can't parse a nil string.");
+        return Qnil;
+    }
+    
+    if (parse_complete_callback != Qnil) {
+        if (context == Qnil) {
+            context = rb_ary_new();
+        }
+        if (chunkedParser == NULL) {
+            // allocate our parser
+            chunkedParser = yajl_alloc(&callbacks, &cfg, NULL, (void *)context);
+        }
+        
+        stat = yajl_parse(chunkedParser, (const unsigned char *)RSTRING_PTR(string), RSTRING_LEN(string));
+        if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) {
+            unsigned char * str = yajl_get_error(chunkedParser, 1, (const unsigned char *)RSTRING_PTR(string), RSTRING_LEN(string));
+            rb_raise(cParseError, "%s", (const char *) str);
+            yajl_free_error(chunkedParser, str);
+        }
+    } else {
+        rb_raise(cParseError, "%s", "The on_parse_complete callback isn't setup, parsing useless.");
+    }
+    
+    if (RARRAY_LEN(context) == 0) {
+        yajl_free(chunkedParser);
+    }
+    
+    return Qnil;
+}
+
+static VALUE t_parse(VALUE self, VALUE io) {
+    yajl_status stat;
+    context = rb_ary_new();
+    
+    // allocate our parser
+    streamParser = yajl_alloc(&callbacks, &cfg, NULL, (void *)context);
+    
+    VALUE parsed = rb_str_new2("");
+    VALUE rbufsize = INT2FIX(readBufferSize);
+    
+    // now parse from the IO
+    while (rb_funcall(io, intern_eof, 0) != Qtrue) {
+        rb_funcall(io, intern_io_read, 2, rbufsize, parsed);
+        
+        stat = yajl_parse(streamParser, (const unsigned char *)RSTRING_PTR(parsed), RSTRING_LEN(parsed));
+        
+        if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) {
+            unsigned char * str = yajl_get_error(streamParser, 1, (const unsigned char *)RSTRING_PTR(parsed), RSTRING_LEN(parsed));
+            rb_raise(cParseError, "%s", (const char *) str);
+            yajl_free_error(streamParser, str);
+            break;
+        }
+    }
+    
+    // parse any remaining buffered data
+    stat = yajl_parse_complete(streamParser);
+    yajl_free(streamParser);
+    
+    if (parse_complete_callback != Qnil) {
+        check_and_fire_callback((void *)context);
+        return Qnil;
+    }
+
+    return rb_ary_pop(context);
+}
+
+static VALUE t_encode(VALUE self, VALUE obj, VALUE io) {
+  yajl_gen_config conf = {0, " "};
+  yajl_gen hand;
+  const unsigned char * buffer;
+  unsigned int len;
+  VALUE outBuff;
+  
+  hand = yajl_gen_alloc(&conf, NULL);
+  encode_part(hand, obj, io);
+  
+  // just make sure we output the remaining buffer
+  yajl_gen_get_buf(hand, &buffer, &len);
+  outBuff = rb_str_new((const char *)buffer, len);
+  rb_io_write(io, outBuff);
+  
+  yajl_gen_clear(hand);
+  yajl_gen_free(hand);
+  return Qnil;
+}
+
+void Init_yajl_ext() {
+    mYajl = rb_define_module("Yajl");
+    
+    mStream = rb_define_module_under(mYajl, "Stream");
+    rb_define_module_function(mStream, "parse", t_parse, 1);
+    rb_define_module_function(mStream, "encode", t_encode, 2);
+    
+    mChunked = rb_define_module_under(mYajl, "Chunked");
+    rb_define_module_function(mChunked, "parse_some", t_parseSome, 1);
+    rb_define_module_function(mChunked, "<<", t_parseSome, 1);
+    rb_define_module_function(mChunked, "on_parse_complete=", t_setParseComplete, 1);
+    
+    VALUE rb_cStandardError = rb_const_get(rb_cObject, rb_intern("StandardError"));
+    cParseError = rb_define_class_under(mYajl, "ParseError", rb_cStandardError);
+    
+    intern_io_read = rb_intern("read");
+    intern_eof = rb_intern("eof?");
+    intern_respond_to = rb_intern("respond_to?");
+    intern_call = rb_intern("call");
+    intern_keys = rb_intern("keys");
+    intern_to_s = rb_intern("to_s");
+}

data/ext/{yajl.h → yajl_ext.h}

@@ -1,5 +1,5 @@
-#include <yajl/yajl_parse.h>
-#include <yajl/yajl_gen.h>
+#include "api/yajl_parse.h"
+#include "api/yajl_gen.h"
 #include <ruby.h>
 
 #define READ_BUFSIZE 4096