yajl-ruby 1.3.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 731d0a5e71e781ff6061f6fda922f73c773ad0ee
-  data.tar.gz: c3a1f37875013458989cf34988bed786d58c7c00
+  metadata.gz: c99601133192fb2ea1f0586ac1b4a83dc465d9a8
+  data.tar.gz: cbfce4619e6c2ccd5b761bf8d6005dbee6ce043e
 SHA512:
-  metadata.gz: 33df12d70c43f4599438ce478755967d174ce403fddc946dd5b5ca2e76723ba064fc10bb1185513927a230bdbbf2ebe479d1666632127bfcd4afe367ea24e646
-  data.tar.gz: 2d78f6564a3c14a17fc55187a1951b5a5fa2b263dff0f47fd8d6d6d377727771989bdcb9b68970b5995fb53449e0d4053cffc96412820525dec184074e68225c
+  metadata.gz: 78c816a3a6368d3b0520d27b5987da10a40478357424e305aa121a6359141ef51c8ee9a3caa4bc3dde159156d5cb6f9af6d5cfd3c576c543b5fd9178ad2993e6
+  data.tar.gz: fc968516c955984c03c184d11a349c82c109adcce8c590a7537bd291a315bb2a8ac0be92ae927b9458e10dee07b564ebf15806bc591b89b0d09df89b5d6ce173

data/.travis.yml

@@ -1,14 +1,9 @@
 language: ruby
 rvm:
-  - 1.8.7
-  - 1.9.2
-  - 1.9.3
   - 2.0.0
   - 2.1
-  - rbx-2
-  - ree
+  - 2.2
+  - 2.3
+  - 2.4.1
   - ruby-head
 before_install: gem install bundler --no-document
-matrix:
-  allow_failures:
-    - rvm: rbx-2

data/ext/yajl/extconf.rb

@@ -1,7 +1,7 @@
 require 'mkmf'
 require 'rbconfig'
 
-$CFLAGS << ' -Wall -funroll-loops'
+$CFLAGS << ' -Wall -funroll-loops -Wno-declaration-after-statement'
 $CFLAGS << ' -Werror-implicit-function-declaration -Wextra -O0 -ggdb3' if ENV['DEBUG']
 
 create_makefile('yajl/yajl')

data/ext/yajl/yajl_encode.c

@@ -59,12 +59,18 @@ yajl_string_encode2(const yajl_print_t print,
                     unsigned int htmlSafe)
 {
     unsigned int beg = 0;
-    unsigned int end = 0;    
+    unsigned int end = 0;
+    unsigned int increment = 0;
     char hexBuf[7];
+    char entityBuffer[7];
     hexBuf[0] = '\\'; hexBuf[1] = 'u'; hexBuf[2] = '0'; hexBuf[3] = '0';
     hexBuf[6] = 0;
 
+    entityBuffer[0] = '\\'; entityBuffer[1] = 'u'; entityBuffer[2] = '2'; entityBuffer[3] = '0';
+    entityBuffer[6] = 0;
+
     while (end < len) {
+        increment = 1;
         const char * escaped = NULL;
         switch (str[end]) {
             case '\r': escaped = "\\r"; break;
@@ -76,10 +82,39 @@ yajl_string_encode2(const yajl_print_t print,
             case '\b': escaped = "\\b"; break;
             case '\t': escaped = "\\t"; break;
             case '/':
-              if (htmlSafe) {
+              if (htmlSafe == 1 || htmlSafe == 2) {
                 escaped = "\\/";
               }
               break;
+            /* Escaping 0xe280a8 0xe280a9 */
+            case 0xe2:
+              if (htmlSafe == 2) {
+                  if (len - end >= 2 && str[end + 1] == 0x80) {
+                      if (str[end + 2] == 0xa8) {
+                          increment = 3;
+                          entityBuffer[4] = '2';
+                          entityBuffer[5] = '8';
+                          escaped = entityBuffer;
+                          break;
+                      }
+
+                      if (str[end + 2] == 0xa9) {
+                          increment = 3;
+                          entityBuffer[4] = '2';
+                          entityBuffer[5] = '9';
+                          escaped = entityBuffer;
+                          break;
+                      }
+                  }
+              }
+            case '<':
+            case '>':
+            case '&':
+              if (htmlSafe == 2) {
+                  CharToHex(str[end], hexBuf + 4);
+                  escaped = hexBuf;
+              }
+              break;
             default:
                 if ((unsigned char) str[end] < 32) {
                     CharToHex(str[end], hexBuf + 4);
@@ -90,7 +125,8 @@ yajl_string_encode2(const yajl_print_t print,
         if (escaped != NULL) {
             print(ctx, (const char *) (str + beg), end - beg);
             print(ctx, escaped, (unsigned int)strlen(escaped));
-            beg = ++end;
+            end += increment;
+            beg = end;
         } else {
             ++end;
         }
@@ -162,8 +198,8 @@ void yajl_string_decode(yajl_buf buf, const unsigned char * str,
                     end+=3;
                     /* check if this is a surrogate */
                     if ((codepoint & 0xFC00) == 0xD800) {
-                        end++;
-                        if (str[end] == '\\' && str[end + 1] == 'u') {
+                        if (end + 2 < len && str[end + 1] == '\\' && str[end + 2] == 'u') {
+                            end++;
                             unsigned int surrogate = 0;
                             hexToDigit(&surrogate, str + end + 2);
                             codepoint =

data/ext/yajl/yajl_ext.c

@@ -22,6 +22,12 @@
 */
 
 #include "yajl_ext.h"
+#include "yajl_lex.h"
+#include "yajl_alloc.h"
+#include "yajl_buf.h"
+#include "yajl_encode.h"
+#include "api/yajl_common.h"
+#include "assert.h"
 
 #define YAJL_RB_TO_JSON                                   \
  VALUE rb_encoder, cls;                                   \
@@ -32,6 +38,25 @@
  }                                                        \
  return rb_yajl_encoder_encode(1, &self, rb_encoder);     \
 
+static void *rb_internal_malloc(void *ctx, unsigned int sz) {
+  return xmalloc(sz);
+}
+
+static void *rb_internal_realloc(void *ctx, void *previous, unsigned int sz) {
+  return xrealloc(previous, sz);
+}
+
+static void rb_internal_free(void *ctx, void *ptr) {
+  xfree(ptr);
+}
+
+static yajl_alloc_funcs rb_alloc_funcs = {
+  rb_internal_malloc,
+  rb_internal_realloc,
+  rb_internal_free,
+  NULL
+};
+
 /* Helpers for building objects */
 static void yajl_check_and_fire_callback(void * ctx) {
     yajl_parser_wrapper * wrapper;
@@ -39,12 +64,12 @@ static void yajl_check_and_fire_callback(void * ctx) {
 
     /* No need to do any of this if the callback isn't even setup */
     if (wrapper->parse_complete_callback != Qnil) {
-        int len = RARRAY_LEN(wrapper->builderStack);
+        long len = RARRAY_LEN(wrapper->builderStack);
         if (len == 1 && wrapper->nestedArrayLevel == 0 && wrapper->nestedHashLevel == 0) {
             rb_funcall(wrapper->parse_complete_callback, intern_call, 1, rb_ary_pop(wrapper->builderStack));
         }
     } else {
-        int len = RARRAY_LEN(wrapper->builderStack);
+        long len = RARRAY_LEN(wrapper->builderStack);
         if (len == 1 && wrapper->nestedArrayLevel == 0 && wrapper->nestedHashLevel == 0) {
             wrapper->objectsFound++;
             if (wrapper->objectsFound > 1) {
@@ -76,7 +101,7 @@ static char *yajl_raise_encode_error_for_status(yajl_gen_status status, VALUE ob
 static void yajl_set_static_value(void * ctx, VALUE val) {
     yajl_parser_wrapper * wrapper;
     VALUE lastEntry, hash;
-    int len;
+    long len;
 
     GetParser((VALUE)ctx, wrapper);
 
@@ -198,7 +223,7 @@ void yajl_encode_part(void * wrapper, VALUE obj, VALUE io) {
         case T_BIGNUM:
             str = rb_funcall(obj, intern_to_s, 0);
             cptr = RSTRING_PTR(str);
-            len = RSTRING_LEN(str);
+            len = (unsigned int)RSTRING_LEN(str);
             if (memcmp(cptr, "NaN", 3) == 0 || memcmp(cptr, "Infinity", 8) == 0 || memcmp(cptr, "-Infinity", 9) == 0) {
                 rb_raise(cEncodeError, "'%s' is an invalid number", cptr);
             }
@@ -206,7 +231,7 @@ void yajl_encode_part(void * wrapper, VALUE obj, VALUE io) {
             break;
         case T_STRING:
             cptr = RSTRING_PTR(obj);
-            len = RSTRING_LEN(obj);
+            len = (unsigned int)RSTRING_LEN(obj);
             CHECK_STATUS(yajl_gen_string(w->encoder, (const unsigned char *)cptr, len));
             break;
         default:
@@ -214,13 +239,13 @@ void yajl_encode_part(void * wrapper, VALUE obj, VALUE io) {
                 str = rb_funcall(obj, intern_to_json, 0);
                 Check_Type(str, T_STRING);
                 cptr = RSTRING_PTR(str);
-                len = RSTRING_LEN(str);
+                len = (unsigned int)RSTRING_LEN(str);
                 CHECK_STATUS(yajl_gen_number(w->encoder, cptr, len));
             } else {
                 str = rb_funcall(obj, intern_to_s, 0);
                 Check_Type(str, T_STRING);
                 cptr = RSTRING_PTR(str);
-                len = RSTRING_LEN(str);
+                len = (unsigned int)RSTRING_LEN(str);
                 CHECK_STATUS(yajl_gen_string(w->encoder, (const unsigned char *)cptr, len));
             }
             break;
@@ -420,7 +445,7 @@ static VALUE rb_yajl_parser_new(int argc, VALUE * argv, VALUE klass) {
     cfg = (yajl_parser_config){allowComments, checkUTF8};
 
     obj = Data_Make_Struct(klass, yajl_parser_wrapper, yajl_parser_wrapper_mark, yajl_parser_wrapper_free, wrapper);
-    wrapper->parser = yajl_alloc(&callbacks, &cfg, NULL, (void *)obj);
+    wrapper->parser = yajl_alloc(&callbacks, &cfg, &rb_alloc_funcs, (void *)obj);
     wrapper->nestedArrayLevel = 0;
     wrapper->nestedHashLevel = 0;
     wrapper->objectsFound = 0;
@@ -489,13 +514,13 @@ static VALUE rb_yajl_parser_parse(int argc, VALUE * argv, VALUE self) {
 
     if (TYPE(input) == T_STRING) {
         cptr = RSTRING_PTR(input);
-        len = RSTRING_LEN(input);
+        len = (unsigned int)RSTRING_LEN(input);
         yajl_parse_chunk((const unsigned char*)cptr, len, wrapper->parser);
     } else if (rb_respond_to(input, intern_io_read)) {
         VALUE parsed = rb_str_new(0, FIX2LONG(rbufsize));
         while (rb_funcall(input, intern_io_read, 2, rbufsize, parsed) != Qnil) {
             cptr = RSTRING_PTR(parsed);
-            len = RSTRING_LEN(parsed);
+            len = (unsigned int)RSTRING_LEN(parsed);
             yajl_parse_chunk((const unsigned char*)cptr, len, wrapper->parser);
         }
     } else {
@@ -535,7 +560,7 @@ static VALUE rb_yajl_parser_parse_chunk(VALUE self, VALUE chunk) {
 
     if (wrapper->parse_complete_callback != Qnil) {
         const char * cptr = RSTRING_PTR(chunk);
-        len = RSTRING_LEN(chunk);
+        len = (unsigned int)RSTRING_LEN(chunk);
         yajl_parse_chunk((const unsigned char*)cptr, len, wrapper->parser);
     } else {
         rb_raise(cParseError, "The on_parse_complete callback isn't setup, parsing useless.");
@@ -560,6 +585,402 @@ static VALUE rb_yajl_parser_set_complete_cb(VALUE self, VALUE callback) {
     return Qnil;
 }
 
+/*
+ * An event stream pulls data off the IO source into the buffer,
+ * then runs the lexer over that stream.
+ */
+struct yajl_event_stream_s {
+    yajl_alloc_funcs *funcs;
+
+    VALUE stream;     // source
+
+    VALUE buffer;
+    unsigned int offset;
+
+    yajl_lexer lexer; // event source
+};
+
+typedef struct yajl_event_stream_s *yajl_event_stream_t;
+
+struct yajl_event_s {
+    yajl_tok token;
+    const char *buf;
+    unsigned int len;
+};
+typedef struct yajl_event_s yajl_event_t;
+
+static yajl_event_t yajl_event_stream_next(yajl_event_stream_t parser, int pop) {
+    assert(parser->stream);
+    assert(parser->buffer);
+
+    while (1) {
+        if (parser->offset >= RSTRING_LEN(parser->buffer)) {
+            //printf("reading offset %d size %ld\n", parser->offset, RSTRING_LEN(parser->buffer));
+
+            // Refill the buffer
+            rb_funcall(parser->stream, intern_io_read, 2, INT2FIX(RSTRING_LEN(parser->buffer)), parser->buffer);
+            if (RSTRING_LEN(parser->buffer) == 0) {
+                yajl_event_t event = {
+                    .token = yajl_tok_eof,
+                };
+                return event;
+            }
+
+            parser->offset = 0;
+        }
+
+        // Try to pull an event off the lexer
+        yajl_event_t event;
+
+        yajl_tok token;
+        if (pop == 0) {
+            //printf("peeking %p %ld %d\n", RSTRING_PTR(parser->buffer), RSTRING_LEN(parser->buffer), parser->offset);
+            token = yajl_lex_peek(parser->lexer, (const unsigned char *)RSTRING_PTR(parser->buffer), (unsigned int)RSTRING_LEN(parser->buffer), parser->offset);
+            //printf("peeked event %d\n", token);
+
+            if (token == yajl_tok_eof) {
+                parser->offset = (unsigned int)RSTRING_LEN(parser->buffer);
+                continue;
+            }
+
+            event.token = token;
+
+            return event;
+        }
+
+        //printf("popping\n");
+        token = yajl_lex_lex(parser->lexer, (const unsigned char *)RSTRING_PTR(parser->buffer), (unsigned int)RSTRING_LEN(parser->buffer), &parser->offset, (const unsigned char **)&event.buf, &event.len);
+        //printf("popped event %d\n", token);
+
+        if (token == yajl_tok_eof) {
+           continue;
+        }
+
+        event.token = token;
+
+        return event;
+    }
+
+    return (yajl_event_t){};
+}
+
+static VALUE rb_yajl_projector_filter_array_subtree(yajl_event_stream_t parser, VALUE schema, yajl_event_t event);
+static VALUE rb_yajl_projector_filter_object_subtree(yajl_event_stream_t parser, VALUE schema, yajl_event_t event);
+static void rb_yajl_projector_ignore_value(yajl_event_stream_t parser);
+static void rb_yajl_projector_ignore_container(yajl_event_stream_t parser);
+static VALUE rb_yajl_projector_build_simple_value(yajl_event_stream_t parser, yajl_event_t event);
+static VALUE rb_yajl_projector_build_string(yajl_event_stream_t parser, yajl_event_t event);
+
+static VALUE rb_yajl_projector_filter(yajl_event_stream_t parser, VALUE schema, yajl_event_t event) {
+    assert(parser->stream);
+
+    switch(event.token) {
+        case yajl_tok_left_brace:
+            return rb_yajl_projector_filter_array_subtree(parser, schema, event);
+            break;
+        case yajl_tok_left_bracket:
+            return rb_yajl_projector_filter_object_subtree(parser, schema, event);
+            break;
+        default:
+            return rb_yajl_projector_build_simple_value(parser, event);
+    }
+}
+
+static VALUE rb_yajl_projector_filter_array_subtree(yajl_event_stream_t parser, VALUE schema, yajl_event_t event) {
+    assert(event.token == yajl_tok_left_brace);
+
+    VALUE ary = rb_ary_new();
+
+    while (1) {
+        event = yajl_event_stream_next(parser, 1);
+
+        if (event.token == yajl_tok_right_brace) {
+            break;
+        }
+
+        VALUE val = rb_yajl_projector_filter(parser, schema, event);
+        rb_ary_push(ary, val);
+
+        event = yajl_event_stream_next(parser, 0);
+        if (event.token == yajl_tok_comma) {
+            event = yajl_event_stream_next(parser, 1);
+            assert(event.token == yajl_tok_comma);
+
+            event = yajl_event_stream_next(parser, 0);
+            if (!(event.token == yajl_tok_string || event.token == yajl_tok_integer || event.token == yajl_tok_double || event.token == yajl_tok_null || event.token == yajl_tok_bool || event.token == yajl_tok_left_bracket || event.token == yajl_tok_left_brace)) {
+                rb_raise(cParseError, "read a comma, expected a value to follow, actually read %s", yajl_tok_name(event.token));
+            }
+        } else if (event.token != yajl_tok_right_brace) {
+            rb_raise(cParseError, "didn't read a comma, expected closing array, actually read %s", yajl_tok_name(event.token));
+        }
+    }
+
+    return ary;
+}
+
+static VALUE rb_yajl_projector_filter_object_subtree(yajl_event_stream_t parser, VALUE schema, yajl_event_t event) {
+    assert(event.token == yajl_tok_left_bracket);
+
+    VALUE hsh = rb_hash_new();
+
+    while (1) {
+        event = yajl_event_stream_next(parser, 1);
+
+        if (event.token == yajl_tok_right_bracket) {
+            break;
+        }
+
+        if (!(event.token == yajl_tok_string || event.token == yajl_tok_string_with_escapes)) {
+            rb_raise(cParseError, "Expected string, unexpected stream event %s", yajl_tok_name(event.token));
+        }
+
+        VALUE key = rb_yajl_projector_build_string(parser, event);
+
+        event = yajl_event_stream_next(parser, 1);
+        if (!(event.token == yajl_tok_colon)) {
+            rb_raise(cParseError, "Expected colon, unexpected stream event %s", yajl_tok_name(event.token));
+        }
+
+        // nil schema means reify the subtree from here on
+        // otherwise if the schema has a key for this we want it
+        int interesting = (schema == Qnil || rb_funcall(schema, rb_intern("key?"), 1, key) == Qtrue);
+        if (!interesting) {
+            rb_yajl_projector_ignore_value(parser);
+            goto peek_comma;
+        }
+
+        yajl_event_t value_event = yajl_event_stream_next(parser, 1);
+
+        VALUE key_schema;
+        if (schema == Qnil) {
+            key_schema = Qnil;
+        } else {
+            key_schema = rb_hash_aref(schema, key);
+        }
+
+        VALUE val = rb_yajl_projector_filter(parser, key_schema, value_event);
+
+        rb_str_freeze(key);
+        rb_hash_aset(hsh, key, val);
+
+    peek_comma:
+
+        event = yajl_event_stream_next(parser, 0);
+        if (event.token == yajl_tok_comma) {
+            event = yajl_event_stream_next(parser, 1);
+            assert(event.token == yajl_tok_comma);
+
+            event = yajl_event_stream_next(parser, 0);
+            if (!(event.token == yajl_tok_string || event.token == yajl_tok_string_with_escapes)) {
+                rb_raise(cParseError, "read a comma, expected a key to follow, actually read %s", yajl_tok_name(event.token));
+            }
+        } else if (event.token != yajl_tok_right_bracket) {
+            rb_raise(cParseError, "read a value without tailing comma, expected closing bracket, actually read %s", yajl_tok_name(event.token));
+        }
+    }
+
+    return hsh;
+}
+
+/*
+# After reading a key if we know we are not interested in the next value,
+    # read and discard all its stream events.
+    #
+    # Values can be simple (string, numeric, boolean, null) or compound (object
+    # or array).
+    #
+    # Returns nothing.
+*/
+static void rb_yajl_projector_ignore_value(yajl_event_stream_t parser) {
+    yajl_event_t value_event = yajl_event_stream_next(parser, 1);
+
+    switch (value_event.token) {
+        case yajl_tok_null:
+        case yajl_tok_bool:
+        case yajl_tok_integer:
+        case yajl_tok_double:
+        case yajl_tok_string:
+        case yajl_tok_string_with_escapes:
+            return;
+        default:
+            break;
+    }
+
+    if (value_event.token == yajl_tok_left_brace || value_event.token == yajl_tok_left_bracket) {
+        rb_yajl_projector_ignore_container(parser);
+        return;
+    }
+
+    rb_raise(cParseError, "unknown value type to ignore %s", yajl_tok_name(value_event.token));
+}
+
+/*
+# Given the start of an array or object, read until the closing event.
+# Object structures can nest and this is considered.
+#
+# Returns nothing.
+*/
+static void rb_yajl_projector_ignore_container(yajl_event_stream_t parser) {
+  int depth = 1;
+
+  while (depth > 0) {
+    yajl_event_t event = yajl_event_stream_next(parser, 1);
+
+    if (event.token == yajl_tok_eof) {
+        return;
+    }
+
+    if (event.token == yajl_tok_left_bracket || event.token == yajl_tok_left_brace) {
+        depth += 1;
+    } else if (event.token == yajl_tok_right_bracket || event.token == yajl_tok_right_brace) {
+        depth -= 1;
+    }
+  }
+}
+
+static VALUE rb_yajl_projector_build_simple_value(yajl_event_stream_t parser, yajl_event_t event) {
+    assert(parser->stream);
+
+    switch (event.token) {
+        case yajl_tok_null:;
+            return Qnil;
+        case yajl_tok_bool:;
+            if (memcmp(event.buf, "true", 4) == 0) {
+                return Qtrue;
+            } else if (memcmp(event.buf, "false", 4) == 0) {
+                return Qfalse;
+            } else {
+                rb_raise(cStandardError, "unknown boolean token %s", event.buf);
+            }
+        case yajl_tok_integer:;
+        case yajl_tok_double:;
+            char *buf = (char *)malloc(event.len + 1);
+            buf[event.len] = 0;
+            memcpy(buf, event.buf, event.len);
+
+            VALUE val;
+            if (memchr(buf, '.', event.len) ||
+                memchr(buf, 'e', event.len) ||
+                memchr(buf, 'E', event.len)) {
+                val = rb_float_new(strtod(buf, NULL));
+            } else {
+                val = rb_cstr2inum(buf, 10);
+            }
+            free(buf);
+
+            return val;
+
+        case yajl_tok_string:;
+        case yajl_tok_string_with_escapes:;
+            return rb_yajl_projector_build_string(parser, event);
+
+        case yajl_tok_eof:;
+            rb_raise(cParseError, "unexpected eof while constructing value");
+
+        case yajl_tok_comma:
+            rb_raise(cParseError, "unexpected comma while constructing value");
+
+        case yajl_tok_colon:
+            rb_raise(cParseError, "unexpected colon while constructing value");
+
+        default:;
+            assert(0);
+    }
+}
+
+static VALUE rb_yajl_projector_build_string(yajl_event_stream_t parser, yajl_event_t event) {
+    switch (event.token) {
+        case yajl_tok_string:; {
+            VALUE str = rb_str_new(event.buf, event.len);
+            rb_enc_associate(str, utf8Encoding);
+
+            rb_encoding *default_internal_enc = rb_default_internal_encoding();
+            if (default_internal_enc) {
+                str = rb_str_export_to_enc(str, default_internal_enc);
+            }
+
+            return str;
+        }
+
+        case yajl_tok_string_with_escapes:; {
+            //printf("decoding string with escapes\n");
+
+            yajl_buf strBuf = yajl_buf_alloc(parser->funcs);
+            yajl_string_decode(strBuf, (const unsigned char *)event.buf, event.len);
+
+            VALUE str = rb_str_new((const char *)yajl_buf_data(strBuf), yajl_buf_len(strBuf));
+            rb_enc_associate(str, utf8Encoding);
+
+            yajl_buf_free(strBuf);
+
+            rb_encoding *default_internal_enc = rb_default_internal_encoding();
+            if (default_internal_enc) {
+                str = rb_str_export_to_enc(str, default_internal_enc);
+            }
+
+            return str;
+        }
+
+        default:; {
+            assert(0);
+        }
+    }
+}
+
+static VALUE rb_protected_yajl_projector_filter(VALUE pointer) {
+    VALUE *args = (VALUE *)pointer;
+    return rb_yajl_projector_filter((struct yajl_event_stream_s *)args[0],
+                                                                  args[1],
+                                                 *(yajl_event_t *)args[2]);
+}
+
+/*
+ * Document-method: project
+ */
+static VALUE rb_yajl_projector_project(VALUE self, VALUE schema) {
+    VALUE stream = rb_iv_get(self, "@stream");
+
+    long buffer_size = FIX2LONG(rb_iv_get(self, "@buffer_size"));
+    VALUE buffer = rb_str_new(0, buffer_size);
+
+    struct yajl_event_stream_s parser = {
+        .funcs = &rb_alloc_funcs,
+
+        .stream = stream,
+
+        .buffer = buffer,
+        .offset = (unsigned int)buffer_size,
+
+        .lexer = yajl_lex_alloc(&rb_alloc_funcs, 0, 1),
+    };
+
+    yajl_event_t event = yajl_event_stream_next(&parser, 1);
+
+    RB_GC_GUARD(stream);
+    RB_GC_GUARD(buffer);
+
+    VALUE result;
+    int state = 0;
+
+    if (event.token == yajl_tok_left_brace || event.token == yajl_tok_left_bracket) {
+        VALUE args[3];
+        args[0] = (VALUE)&parser;
+        args[1] = schema;
+        args[2] = (VALUE)&event;
+        result = rb_protect(rb_protected_yajl_projector_filter,
+                            (VALUE)args,
+                            &state);
+    } else {
+        yajl_lex_free(parser.lexer);
+        rb_raise(cParseError, "expected left bracket or brace, actually read %s", yajl_tok_name(event.token));
+    }
+
+    yajl_lex_free(parser.lexer);
+    if (state) rb_jump_tag(state);
+
+    return result;
+}
+
 /*
  * Document-class: Yajl::Encoder
  *
@@ -609,9 +1030,14 @@ static VALUE rb_yajl_encoder_new(int argc, VALUE * argv, VALUE klass) {
                 actualIndent = indentString;
             }
         }
+
         if (rb_hash_aref(opts, sym_html_safe) == Qtrue) {
           htmlSafe = 1;
         }
+
+        if (rb_hash_aref(opts, sym_entities) == Qtrue) {
+          htmlSafe = 2;
+        }
     }
     if (!indentString) {
       indentString = defaultIndentString;
@@ -620,7 +1046,7 @@ static VALUE rb_yajl_encoder_new(int argc, VALUE * argv, VALUE klass) {
 
     obj = Data_Make_Struct(klass, yajl_encoder_wrapper, yajl_encoder_wrapper_mark, yajl_encoder_wrapper_free, wrapper);
     wrapper->indentString = actualIndent;
-    wrapper->encoder = yajl_gen_alloc(&cfg, NULL);
+    wrapper->encoder = yajl_gen_alloc(&cfg, &rb_alloc_funcs);
     wrapper->on_progress_callback = Qnil;
     if (opts != Qnil && rb_funcall(opts, intern_has_key, 1, sym_terminator) == Qtrue) {
         wrapper->terminator = rb_hash_aref(opts, sym_terminator);
@@ -900,6 +1326,7 @@ void Init_yajl() {
 
     cParseError = rb_define_class_under(mYajl, "ParseError", rb_eStandardError);
     cEncodeError = rb_define_class_under(mYajl, "EncodeError", rb_eStandardError);
+    cStandardError = rb_const_get(rb_cObject, rb_intern("StandardError"));
 
     cParser = rb_define_class_under(mYajl, "Parser", rb_cObject);
     rb_define_singleton_method(cParser, "new", rb_yajl_parser_new, -1);
@@ -909,6 +1336,9 @@ void Init_yajl() {
     rb_define_method(cParser, "<<", rb_yajl_parser_parse_chunk, 1);
     rb_define_method(cParser, "on_parse_complete=", rb_yajl_parser_set_complete_cb, 1);
 
+    cProjector = rb_define_class_under(mYajl, "Projector", rb_cObject);
+    rb_define_method(cProjector, "project", rb_yajl_projector_project, 1);
+
     cEncoder = rb_define_class_under(mYajl, "Encoder", rb_cObject);
     rb_define_singleton_method(cEncoder, "new", rb_yajl_encoder_new, -1);
     rb_define_method(cEncoder, "initialize", rb_yajl_encoder_init, -1);
@@ -931,6 +1361,7 @@ void Init_yajl() {
     sym_pretty = ID2SYM(rb_intern("pretty"));
     sym_indent = ID2SYM(rb_intern("indent"));
     sym_html_safe = ID2SYM(rb_intern("html_safe"));
+    sym_entities = ID2SYM(rb_intern("entities"));
     sym_terminator = ID2SYM(rb_intern("terminator"));
     sym_symbolize_keys = ID2SYM(rb_intern("symbolize_keys"));
     sym_symbolize_names = ID2SYM(rb_intern("symbolize_names"));