oj 2.9.0 → 2.9.1

data/ext/oj/reader.c

@@ -0,0 +1,244 @@
+/* reader.c
+ * Copyright (c) 2011, Peter Ohler
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *  - Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 
+ *  - Neither the name of Peter Ohler nor the names of its contributors may be
+ *    used to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdlib.h>
+#include <errno.h>
+#include <stdio.h>
+#include <strings.h>
+#include <sys/types.h>
+#if NEEDS_UIO
+#include <sys/uio.h>	
+#endif
+#include <unistd.h>
+#include <time.h>
+
+#include "ruby.h"
+#include "oj.h"
+#include "reader.h"
+
+#define BUF_PAD	4
+
+static VALUE		rescue_cb(VALUE rdr, VALUE err);
+static VALUE		io_cb(VALUE rdr);
+static VALUE		partial_io_cb(VALUE rdr);
+static int		read_from_io(Reader reader);
+static int		read_from_fd(Reader reader);
+static int		read_from_io_partial(Reader reader);
+static int		read_from_str(Reader reader);
+
+void
+oj_reader_init(Reader reader, VALUE io, int fd) {
+    reader->head = reader->base;
+    *((char*)reader->head) = '\0';
+    reader->end = reader->head + sizeof(reader->base) - BUF_PAD;
+    reader->tail = reader->head;
+    reader->read_end = reader->head;
+    reader->pro = 0;
+    reader->str = 0;
+    reader->line = 1;
+    reader->col = 0;
+    reader->free_head = 0;
+
+    if (0 != fd) {
+	reader->read_func = read_from_fd;
+	reader->fd = fd;
+    } else if (rb_cString == rb_obj_class(io)) {
+	reader->read_func = read_from_str;
+	reader->in_str = StringValuePtr(io);
+	reader->head = (char*)reader->in_str;
+	reader->tail = reader->head;
+	reader->read_end = reader->head + RSTRING_LEN(io);
+    } else if (oj_stringio_class == rb_obj_class(io)) {
+	VALUE	s = rb_funcall2(io, oj_string_id, 0, 0);
+
+	reader->read_func = read_from_str;
+	reader->in_str = StringValuePtr(s);
+	reader->head = (char*)reader->in_str;
+	reader->tail = reader->head;
+	reader->read_end = reader->head + RSTRING_LEN(s);
+    } else if (rb_respond_to(io, oj_readpartial_id)) {
+	VALUE	rfd;
+
+	if (rb_respond_to(io, oj_fileno_id) && Qnil != (rfd = rb_funcall(io, oj_fileno_id, 0))) {
+	    reader->read_func = read_from_fd;
+	    reader->fd = FIX2INT(rfd);
+	} else {
+	    reader->read_func = read_from_io_partial;
+	    reader->io = io;
+	}
+    } else if (rb_respond_to(io, oj_read_id)) {
+	VALUE	rfd;
+
+	if (rb_respond_to(io, oj_fileno_id) && Qnil != (rfd = rb_funcall(io, oj_fileno_id, 0))) {
+	    reader->read_func = read_from_fd;
+	    reader->fd = FIX2INT(rfd);
+	} else {
+	    reader->read_func = read_from_io;
+	    reader->io = io;
+	}
+    } else {
+	rb_raise(rb_eException, "parser io argument must respond to readpartial() or read().\n");
+    }
+}
+
+int
+oj_reader_read(Reader reader) {
+    int		err;
+    size_t	shift = 0;
+    
+    if (0 == reader->read_func) {
+	return -1;
+    }
+    // if there is not much room to read into, shift or realloc a larger buffer.
+    if (reader->head < reader->tail && 4096 > reader->end - reader->tail) {
+	if (0 == reader->pro) {
+	    shift = reader->tail - reader->head;
+	} else {
+	    shift = reader->pro - reader->head - 1; // leave one character so we can backup one
+	}
+	if (0 >= shift) { /* no space left so allocate more */
+	    char	*old = reader->head;
+	    size_t	size = reader->end - reader->head + BUF_PAD;
+	
+	    if (reader->head == reader->base) {
+		reader->head = ALLOC_N(char, size * 2);
+		memcpy(reader->head, old, size);
+	    } else {
+		REALLOC_N(reader->head, char, size * 2);
+	    }
+	    reader->free_head = 1;
+	    reader->end = reader->head + size * 2 - BUF_PAD;
+	    reader->tail = reader->head + (reader->tail - old);
+	    reader->read_end = reader->head + (reader->read_end - old);
+	    if (0 != reader->pro) {
+		reader->pro = reader->head + (reader->pro - old);
+	    }
+	    if (0 != reader->str) {
+		reader->str = reader->head + (reader->str - old);
+	    }
+	} else {
+	    memmove(reader->head, reader->head + shift, reader->read_end - (reader->head + shift));
+	    reader->tail -= shift;
+	    reader->read_end -= shift;
+	    if (0 != reader->pro) {
+		reader->pro -= shift;
+	    }
+	    if (0 != reader->str) {
+		reader->str -= shift;
+	    }
+	}
+    }
+    err = reader->read_func(reader);
+    *reader->read_end = '\0';
+
+    return err;
+}
+
+static VALUE
+rescue_cb(VALUE rbuf, VALUE err) {
+#if (defined(RUBINIUS_RUBY) || (1 == RUBY_VERSION_MAJOR && 8 == RUBY_VERSION_MINOR))
+    if (rb_obj_class(err) != rb_eTypeError) {
+#else
+    if (rb_obj_class(err) != rb_eEOFError) {
+#endif
+	Reader	reader = (Reader)rbuf;
+
+	rb_raise(err, "at line %d, column %d\n", reader->line, reader->col);
+    }
+    return Qfalse;
+}
+
+static VALUE
+partial_io_cb(VALUE rbuf) {
+    Reader	reader = (Reader)rbuf;
+    VALUE	args[1];
+    VALUE	rstr;
+    char	*str;
+    size_t	cnt;
+
+    args[0] = ULONG2NUM(reader->end - reader->tail);
+    rstr = rb_funcall2(reader->io, oj_readpartial_id, 1, args);
+    str = StringValuePtr(rstr);
+    cnt = strlen(str);
+    //printf("*** read %lu bytes, str: '%s'\n", cnt, str);
+    strcpy(reader->tail, str);
+    reader->read_end = reader->tail + cnt;
+
+    return Qtrue;
+}
+
+static VALUE
+io_cb(VALUE rbuf) {
+    Reader	reader = (Reader)rbuf;
+    VALUE	args[1];
+    VALUE	rstr;
+    char	*str;
+    size_t	cnt;
+
+    args[0] = ULONG2NUM(reader->end - reader->tail);
+    rstr = rb_funcall2(reader->io, oj_read_id, 1, args);
+    str = StringValuePtr(rstr);
+    cnt = strlen(str);
+    //printf("*** read %lu bytes, str: '%s'\n", cnt, str);
+    strcpy(reader->tail, str);
+    reader->read_end = reader->tail + cnt;
+
+    return Qtrue;
+}
+
+static int
+read_from_io_partial(Reader reader) {
+    return (Qfalse == rb_rescue(partial_io_cb, (VALUE)reader, rescue_cb, (VALUE)reader));
+}
+
+static int
+read_from_io(Reader reader) {
+    return (Qfalse == rb_rescue(io_cb, (VALUE)reader, rescue_cb, (VALUE)reader));
+}
+
+static int
+read_from_fd(Reader reader) {
+    ssize_t	cnt;
+    size_t	max = reader->end - reader->tail;
+
+    cnt = read(reader->fd, reader->tail, max);
+    if (cnt < 0) {
+	return -1;
+    } else if (0 != cnt) {
+	reader->read_end = reader->tail + cnt;
+    }
+    return 0;
+}
+
+// This is only called when the end of the string is reached so just return -1.
+static int
+read_from_str(Reader reader) {
+    return -1;
+}

data/ext/oj/reader.h

@@ -0,0 +1,173 @@
+/* reader.h
+ * Copyright (c) 2011, Peter Ohler
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *  - Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 
+ *  - Neither the name of Peter Ohler nor the names of its contributors may be
+ *    used to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __OJ_READER_H__
+#define __OJ_READER_H__
+
+typedef struct _Reader {
+    char	base[0x00001000];
+    char	*head;
+    char	*end;
+    char	*tail;
+    char	*read_end;	/* one past last character read */
+    char	*pro;		/* protection start, buffer can not slide past this point */
+    char	*str;		/* start of current string being read */
+    int		line;
+    int		col;
+    int		free_head;
+    int		(*read_func)(struct _Reader *reader);
+    union {
+	int		fd;
+	VALUE		io;
+	const char	*in_str;
+    };
+} *Reader;
+
+extern void	oj_reader_init(Reader reader, VALUE io, int fd);
+extern int	oj_reader_read(Reader reader);
+
+static inline char
+reader_get(Reader reader) {
+    //printf("*** drive get from '%s'  from start: %ld	buf: %p	 from read_end: %ld\n", reader->tail, reader->tail - reader->head, reader->head, reader->read_end - reader->tail);
+    if (reader->read_end <= reader->tail) {
+	if (0 != oj_reader_read(reader)) {
+	    return '\0';
+	}
+    }
+    if ('\n' == *reader->tail) {
+	reader->line++;
+	reader->col = 0;
+    }
+    reader->col++;
+    
+    return *reader->tail++;
+}
+
+static inline void
+reader_backup(Reader reader) {
+    reader->tail--;
+    reader->col--;
+    if (0 >= reader->col) {
+	reader->line--;
+	// allow col to be negative since we never backup twice in a row
+    }
+}
+
+static inline void
+reader_protect(Reader reader) {
+    reader->pro = reader->tail;
+    reader->str = reader->tail; // can't have str before pro
+}
+
+static inline void
+reader_release(Reader reader) {
+    reader->pro = 0;
+}
+
+/* Starts by reading a character so it is safe to use with an empty or
+ * compacted buffer.
+ */
+static inline char
+reader_next_non_white(Reader reader) {
+    char	c;
+
+    while ('\0' != (c = reader_get(reader))) {
+	switch(c) {
+	case ' ':
+	case '\t':
+	case '\f':
+	case '\n':
+	case '\r':
+	    break;
+	default:
+	    return c;
+	}
+    }
+    return '\0';
+}
+
+/* Starts by reading a character so it is safe to use with an empty or
+ * compacted buffer.
+ */
+static inline char
+reader_next_white(Reader reader) {
+    char	c;
+
+    while ('\0' != (c = reader_get(reader))) {
+	switch(c) {
+	case ' ':
+	case '\t':
+	case '\f':
+	case '\n':
+	case '\r':
+	case '\0':
+	    return c;
+	default:
+	    break;
+	}
+    }
+    return '\0';
+}
+
+static inline int
+reader_expect(Reader reader, const char *s) {
+    for (; '\0' != *s; s++) {
+	if (reader_get(reader) != *s) {
+	    return -1;
+	}
+    }
+    return 0;
+}
+
+static inline void
+reader_cleanup(Reader reader) {
+    if (reader->free_head && 0 != reader->head) {
+	xfree(reader->head);
+	reader->head = 0;
+	reader->free_head = 0;
+    }
+}
+
+static inline int
+is_white(char c) {
+    switch(c) {
+    case ' ':
+    case '\t':
+    case '\f':
+    case '\n':
+    case '\r':
+	return 1;
+    default:
+	break;
+    }
+    return 0;
+}
+
+#endif /* __OJ_READER_H__ */

data/ext/oj/sparse.c

@@ -0,0 +1,803 @@
+/* parse.c
+ * Copyright (c) 2013, Peter Ohler
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *  - Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ * 
+ *  - Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 
+ *  - Neither the name of Peter Ohler nor the names of its contributors may be
+ *    used to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <math.h>
+
+#include "oj.h"
+#include "parse.h"
+#include "buf.h"
+#include "val_stack.h"
+
+// Workaround in case INFINITY is not defined in math.h or if the OS is CentOS
+#define OJ_INFINITY	(1.0/0.0)
+
+#ifdef RUBINIUS_RUBY
+#define NUM_MAX		0x07FFFFFF
+#else
+#define NUM_MAX		(FIXNUM_MAX >> 8)
+#endif
+#define EXP_MAX		1023
+#define DEC_MAX		14
+
+static void
+skip_comment(ParseInfo pi) {
+    char	c = reader_get(&pi->rd);
+
+    if ('*' == c) {
+	while ('\0' != (c = reader_get(&pi->rd))) {
+	    if ('*' == c) {
+		c = reader_get(&pi->rd);
+		if ('/' == c) {
+		    return;
+		}
+	    }
+	}
+    } else if ('/' == c) {
+	while ('\0' != (c = reader_get(&pi->rd))) {
+	    switch (c) {
+	    case '\n':
+	    case '\r':
+	    case '\f':
+	    case '\0':
+		return;
+	    default:
+		break;
+	    }
+	}
+    } else {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid comment format");
+    }
+    if ('\0' == c) {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "comment not terminated");
+	return;
+    }
+}
+
+static void
+add_value(ParseInfo pi, VALUE rval) {
+    Val	parent = stack_peek(&pi->stack);
+
+    if (0 == parent) { // simple add
+	pi->add_value(pi, rval);
+    } else {
+	switch (parent->next) {
+	case NEXT_ARRAY_NEW:
+	case NEXT_ARRAY_ELEMENT:
+	    pi->array_append_value(pi, rval);
+	    parent->next = NEXT_ARRAY_COMMA;
+	    break;
+	case NEXT_HASH_VALUE:
+	    pi->hash_set_value(pi, parent->key, parent->klen, rval);
+	    if (parent->kalloc) {
+		xfree((char*)parent->key);
+	    }
+	    parent->key = 0;
+	    parent->kalloc = 0;
+	    parent->next = NEXT_HASH_COMMA;
+	    break;
+	case NEXT_HASH_NEW:
+	case NEXT_HASH_KEY:
+	case NEXT_HASH_COMMA:
+	case NEXT_NONE:
+	case NEXT_ARRAY_COMMA:
+	case NEXT_HASH_COLON:
+	default:
+	    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected %s", oj_stack_next_string(parent->next));
+	    break;
+	}
+    }
+}
+
+static void
+add_num_value(ParseInfo pi, NumInfo ni) {
+    Val	parent = stack_peek(&pi->stack);
+
+    if (0 == parent) {
+	pi->add_num(pi, ni);
+    } else {
+	switch (parent->next) {
+	case NEXT_ARRAY_NEW:
+	case NEXT_ARRAY_ELEMENT:
+	    pi->array_append_num(pi, ni);
+	    parent->next = NEXT_ARRAY_COMMA;
+	    break;
+	case NEXT_HASH_VALUE:
+	    pi->hash_set_num(pi, parent->key, parent->klen, ni);
+	    if (parent->kalloc) {
+		xfree((char*)parent->key);
+	    }
+	    parent->key = 0;
+	    parent->kalloc = 0;
+	    parent->next = NEXT_HASH_COMMA;
+	    break;
+	default:
+	    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected %s", oj_stack_next_string(parent->next));
+	    break;
+	}
+    }
+}
+
+static void
+read_true(ParseInfo pi) {
+    if (0 == reader_expect(&pi->rd, "rue")) {
+	add_value(pi, Qtrue);
+    } else {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected true");
+    }
+}
+
+static void
+read_false(ParseInfo pi) {
+    if (0 == reader_expect(&pi->rd, "alse")) {
+	add_value(pi, Qfalse);
+    } else {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected false");
+    }
+}
+
+static uint32_t
+read_hex(ParseInfo pi) {
+    uint32_t	b = 0;
+    int		i;
+    char	c;
+
+    for (i = 0; i < 4; i++) {
+	c = reader_get(&pi->rd);
+	b = b << 4;
+	if ('0' <= c && c <= '9') {
+	    b += c - '0';
+	} else if ('A' <= c && c <= 'F') {
+	    b += c - 'A' + 10;
+	} else if ('a' <= c && c <= 'f') {
+	    b += c - 'a' + 10;
+	} else {
+	    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid hex character");
+	    return 0;
+	}
+    }
+    return b;
+}
+
+static void
+unicode_to_chars(ParseInfo pi, Buf buf, uint32_t code) {
+    if (0x0000007F >= code) {
+	buf_append(buf, (char)code);
+    } else if (0x000007FF >= code) {
+	buf_append(buf, 0xC0 | (code >> 6));
+	buf_append(buf, 0x80 | (0x3F & code));
+    } else if (0x0000FFFF >= code) {
+	buf_append(buf, 0xE0 | (code >> 12));
+	buf_append(buf, 0x80 | ((code >> 6) & 0x3F));
+	buf_append(buf, 0x80 | (0x3F & code));
+    } else if (0x001FFFFF >= code) {
+	buf_append(buf, 0xF0 | (code >> 18));
+	buf_append(buf, 0x80 | ((code >> 12) & 0x3F));
+	buf_append(buf, 0x80 | ((code >> 6) & 0x3F));
+	buf_append(buf, 0x80 | (0x3F & code));
+    } else if (0x03FFFFFF >= code) {
+	buf_append(buf, 0xF8 | (code >> 24));
+	buf_append(buf, 0x80 | ((code >> 18) & 0x3F));
+	buf_append(buf, 0x80 | ((code >> 12) & 0x3F));
+	buf_append(buf, 0x80 | ((code >> 6) & 0x3F));
+	buf_append(buf, 0x80 | (0x3F & code));
+    } else if (0x7FFFFFFF >= code) {
+	buf_append(buf, 0xFC | (code >> 30));
+	buf_append(buf, 0x80 | ((code >> 24) & 0x3F));
+	buf_append(buf, 0x80 | ((code >> 18) & 0x3F));
+	buf_append(buf, 0x80 | ((code >> 12) & 0x3F));
+	buf_append(buf, 0x80 | ((code >> 6) & 0x3F));
+	buf_append(buf, 0x80 | (0x3F & code));
+    } else {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid Unicode character");
+    }
+}
+
+// entered at backslash
+static void
+read_escaped_str(ParseInfo pi) {
+    struct _Buf	buf;
+    char	c;
+    uint32_t	code;
+    Val		parent = stack_peek(&pi->stack);
+
+    buf_init(&buf);
+    if (pi->rd.str < pi->rd.tail) {
+	buf_append_string(&buf, pi->rd.str, pi->rd.tail - pi->rd.str);
+    }
+    while ('\"' != (c = reader_get(&pi->rd))) {
+	if ('\0' == c) {
+	    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "quoted string not terminated");
+	    buf_cleanup(&buf);
+	    return;
+	} else if ('\\' == c) {
+	    c = reader_get(&pi->rd);
+	    switch (c) {
+	    case 'n':	buf_append(&buf, '\n');	break;
+	    case 'r':	buf_append(&buf, '\r');	break;
+	    case 't':	buf_append(&buf, '\t');	break;
+	    case 'f':	buf_append(&buf, '\f');	break;
+	    case 'b':	buf_append(&buf, '\b');	break;
+	    case '"':	buf_append(&buf, '"');	break;
+	    case '/':	buf_append(&buf, '/');	break;
+	    case '\\':	buf_append(&buf, '\\');	break;
+	    case 'u':
+		if (0 == (code = read_hex(pi)) && err_has(&pi->err)) {
+		    buf_cleanup(&buf);
+		    return;
+		}
+		if (0x0000D800 <= code && code <= 0x0000DFFF) {
+		    uint32_t	c1 = (code - 0x0000D800) & 0x000003FF;
+		    uint32_t	c2;
+		    char	ch2;
+
+		    c = reader_get(&pi->rd);
+		    ch2 = reader_get(&pi->rd);
+		    if ('\\' != c || 'u' != ch2) {
+			oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid escaped character");
+			buf_cleanup(&buf);
+			return;
+		    }
+		    if (0 == (c2 = read_hex(pi)) && err_has(&pi->err)) {
+			buf_cleanup(&buf);
+			return;
+		    }
+		    c2 = (c2 - 0x0000DC00) & 0x000003FF;
+		    code = ((c1 << 10) | c2) + 0x00010000;
+		}
+		unicode_to_chars(pi, &buf, code);
+		if (err_has(&pi->err)) {
+		    buf_cleanup(&buf);
+		    return;
+		}
+		break;
+	    default:
+		oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid escaped character");
+		buf_cleanup(&buf);
+		return;
+	    }
+	} else {
+	    buf_append(&buf, c);
+	}
+    }
+    if (0 == parent) {
+	pi->add_cstr(pi, buf.head, buf_len(&buf), pi->rd.str);
+    } else {
+	switch (parent->next) {
+	case NEXT_ARRAY_NEW:
+	case NEXT_ARRAY_ELEMENT:
+	    pi->array_append_cstr(pi, buf.head, buf_len(&buf), pi->rd.str);
+	    parent->next = NEXT_ARRAY_COMMA;
+	    break;
+	case NEXT_HASH_NEW:
+	case NEXT_HASH_KEY:
+	    parent->key = strdup(buf.head);
+	    parent->klen = buf_len(&buf);
+	    parent->k1 = *pi->rd.str;
+	    parent->next = NEXT_HASH_COLON;
+	    break;
+	case NEXT_HASH_VALUE:
+	    pi->hash_set_cstr(pi, parent->key, parent->klen, buf.head, buf_len(&buf), pi->rd.str);
+	    if (parent->kalloc) {
+		xfree((char*)parent->key);
+	    }
+	    parent->key = 0;
+	    parent->kalloc = 0;
+	    parent->next = NEXT_HASH_COMMA;
+	    break;
+	case NEXT_HASH_COMMA:
+	case NEXT_NONE:
+	case NEXT_ARRAY_COMMA:
+	case NEXT_HASH_COLON:
+	default:
+	    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected %s, not a string", oj_stack_next_string(parent->next));
+	    break;
+	}
+    }
+    buf_cleanup(&buf);
+}
+
+static void
+read_str(ParseInfo pi) {
+    Val		parent = stack_peek(&pi->stack);
+    char	c;
+
+    reader_protect(&pi->rd);
+    while ('\"' != (c = reader_get(&pi->rd))) {
+	if ('\0' == c) {
+	    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "quoted string not terminated");
+	    return;
+	} else if ('\\' == c) {
+	    reader_backup(&pi->rd);
+	    read_escaped_str(pi);
+	    reader_release(&pi->rd);
+	    return;
+	}
+    }
+    if (0 == parent) { // simple add
+	pi->add_cstr(pi, pi->rd.str, pi->rd.tail - pi->rd.str - 1, pi->rd.str);
+    } else {
+	switch (parent->next) {
+	case NEXT_ARRAY_NEW:
+	case NEXT_ARRAY_ELEMENT:
+	    pi->array_append_cstr(pi, pi->rd.str, pi->rd.tail - pi->rd.str - 1, pi->rd.str);
+	    parent->next = NEXT_ARRAY_COMMA;
+	    break;
+	case NEXT_HASH_NEW:
+	case NEXT_HASH_KEY:
+	    parent->klen = pi->rd.tail - pi->rd.str - 1;
+	    if (sizeof(parent->karray) <= parent->klen) {
+		parent->key = strndup(pi->rd.str, parent->klen);
+		parent->kalloc = 1;
+	    } else {
+		memcpy(parent->karray, pi->rd.str, parent->klen);
+		parent->karray[parent->klen] = '\0';
+		parent->key = parent->karray;
+		parent->kalloc = 0;
+	    }
+	    parent->k1 = *pi->rd.str;
+	    parent->next = NEXT_HASH_COLON;
+	    break;
+	case NEXT_HASH_VALUE:
+	    pi->hash_set_cstr(pi, parent->key, parent->klen, pi->rd.str, pi->rd.tail - pi->rd.str - 1, pi->rd.str);
+	    if (parent->kalloc) {
+		xfree((char*)parent->key);
+	    }
+	    parent->key = 0;
+	    parent->kalloc = 0;
+	    parent->next = NEXT_HASH_COMMA;
+	    break;
+	case NEXT_HASH_COMMA:
+	case NEXT_NONE:
+	case NEXT_ARRAY_COMMA:
+	case NEXT_HASH_COLON:
+	default:
+	    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected %s, not a string", oj_stack_next_string(parent->next));
+	    break;
+	}
+    }
+    reader_release(&pi->rd);
+}
+
+static void
+read_num(ParseInfo pi) {
+    struct _NumInfo	ni;
+    int			zero_cnt = 0;
+    char		c;
+
+    reader_protect(&pi->rd);
+    ni.i = 0;
+    ni.num = 0;
+    ni.div = 1;
+    ni.len = 0;
+    ni.exp = 0;
+    ni.dec_cnt = 0;
+    ni.big = 0;
+    ni.infinity = 0;
+    ni.nan = 0;
+    ni.neg = 0;
+    ni.no_big = (FloatDec == pi->options.bigdec_load);
+    c = reader_get(&pi->rd);
+    if ('-' == c) {
+	c = reader_get(&pi->rd);
+	ni.neg = 1;
+    } else if ('+' == c) {
+	c = reader_get(&pi->rd);
+    }
+    if ('I' == c) {
+	if (0 != reader_expect(&pi->rd, "nfinity")) {
+	    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number or other value");
+	    return;
+	}
+	ni.infinity = 1;
+    } else {
+	for (; '0' <= c && c <= '9'; c = reader_get(&pi->rd)) {
+	    ni.dec_cnt++;
+	    if (ni.big) {
+		ni.big++;
+	    } else {
+		int	d = (c - '0');
+
+		if (0 == d) {
+		    zero_cnt++;
+		} else {
+		    zero_cnt = 0;
+		}
+		ni.i = ni.i * 10 + d;
+		if (LONG_MAX <= ni.i || DEC_MAX < ni.dec_cnt - zero_cnt) {
+		    ni.big = 1;
+		}
+	    }
+	}
+	if ('.' == c) {
+	    c = reader_get(&pi->rd);
+	    for (; '0' <= c && c <= '9'; c = reader_get(&pi->rd)) {
+		int	d = (c - '0');
+
+		if (0 == d) {
+		    zero_cnt++;
+		} else {
+		    zero_cnt = 0;
+		}
+		ni.dec_cnt++;
+		ni.num = ni.num * 10 + d;
+		ni.div *= 10;
+		if (LONG_MAX <= ni.div || DEC_MAX < ni.dec_cnt - zero_cnt) {
+		    ni.big = 1;
+		}
+	    }
+	}
+	if ('e' == c || 'E' == c) {
+	    int	eneg = 0;
+
+	    c = reader_get(&pi->rd);
+	    if ('-' == c) {
+		c = reader_get(&pi->rd);
+		eneg = 1;
+	    } else if ('+' == c) {
+		c = reader_get(&pi->rd);
+	    }
+	    for (; '0' <= c && c <= '9'; c = reader_get(&pi->rd)) {
+		ni.exp = ni.exp * 10 + (c - '0');
+		if (EXP_MAX <= ni.exp) {
+		    ni.big = 1;
+		}
+	    }
+	    if (eneg) {
+		ni.exp = -ni.exp;
+	    }
+	}
+	ni.dec_cnt -= zero_cnt;
+	ni.len = pi->rd.tail - pi->rd.str;
+	reader_backup(&pi->rd);
+    }
+    if (BigDec == pi->options.bigdec_load) {
+	ni.big = 1;
+    }
+    ni.str = pi->rd.str;
+    ni.len = pi->rd.tail - pi->rd.str;
+    add_num_value(pi, &ni);
+    reader_release(&pi->rd);
+}
+
+static void
+read_nan(ParseInfo pi) {
+    struct _NumInfo	ni;
+    char		c;
+
+    ni.str = pi->rd.str;
+    ni.i = 0;
+    ni.num = 0;
+    ni.div = 1;
+    ni.len = 0;
+    ni.exp = 0;
+    ni.dec_cnt = 0;
+    ni.big = 0;
+    ni.infinity = 0;
+    ni.nan = 1;
+    ni.neg = 0;
+    ni.no_big = (FloatDec == pi->options.bigdec_load);
+
+    if ('a' != reader_get(&pi->rd) ||
+	('N' != (c = reader_get(&pi->rd)) && 'n' != c)) {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number or other value");
+	return;
+    }
+    if (BigDec == pi->options.bigdec_load) {
+	ni.big = 1;
+    }
+    add_num_value(pi, &ni);
+}
+
+static void
+array_start(ParseInfo pi) {
+    VALUE	v = pi->start_array(pi);
+
+    stack_push(&pi->stack, v, NEXT_ARRAY_NEW);
+}
+
+static void
+array_end(ParseInfo pi) {
+    Val	array = stack_pop(&pi->stack);
+
+    if (0 == array) {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected array close");
+    } else if (NEXT_ARRAY_COMMA != array->next && NEXT_ARRAY_NEW != array->next) {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected %s, not an array close", oj_stack_next_string(array->next));
+    } else {
+	pi->end_array(pi);
+	add_value(pi, array->val);
+    }
+}
+
+static void
+hash_start(ParseInfo pi) {
+    volatile VALUE	v = pi->start_hash(pi);
+
+    stack_push(&pi->stack, v, NEXT_HASH_NEW);
+}
+
+static void
+hash_end(ParseInfo pi) {
+    volatile Val	hash = stack_peek(&pi->stack);
+
+    // leave hash on stack until just before 
+    if (0 == hash) {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected hash close");
+    } else if (NEXT_HASH_COMMA != hash->next && NEXT_HASH_NEW != hash->next) {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected %s, not a hash close", oj_stack_next_string(hash->next));
+    } else {
+	pi->end_hash(pi);
+	stack_pop(&pi->stack);
+	add_value(pi, hash->val);
+    }
+}
+
+static void
+comma(ParseInfo pi) {
+    Val	parent = stack_peek(&pi->stack);
+
+    if (0 == parent) {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected comma");
+    } else if (NEXT_ARRAY_COMMA == parent->next) {
+	parent->next = NEXT_ARRAY_ELEMENT;
+    } else if (NEXT_HASH_COMMA == parent->next) {
+	parent->next = NEXT_HASH_KEY;
+    } else {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected comma");
+    }
+}
+
+static void
+colon(ParseInfo pi) {
+    Val	parent = stack_peek(&pi->stack);
+
+    if (0 != parent && NEXT_HASH_COLON == parent->next) {
+	parent->next = NEXT_HASH_VALUE;
+    } else {
+	oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected colon");
+    }
+}
+
+void
+oj_sparse2(ParseInfo pi) {
+    char	c;
+
+    err_init(&pi->err);
+    while (1) {
+	c = reader_next_non_white(&pi->rd);
+	switch (c) {
+	case '{':
+	    hash_start(pi);
+	    break;
+	case '}':
+	    hash_end(pi);
+	    break;
+	case ':':
+	    colon(pi);
+	    break;
+	case '[':
+	    array_start(pi);
+	    break;
+	case ']':
+	    array_end(pi);
+	    break;
+	case ',':
+	    comma(pi);
+	    break;
+	case '"':
+	    read_str(pi);
+	    break;
+	case '+':
+	case '-':
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	case 'I':
+	    reader_backup(&pi->rd);
+	    read_num(pi);
+	    break;
+	case 'N':
+	    read_nan(pi);
+	    break;
+	case 't':
+	    read_true(pi);
+	    break;
+	case 'f':
+	    read_false(pi);
+	    break;
+	case 'n':
+	    c = reader_get(&pi->rd);
+	    if ('u' == c) {
+		if (0 == reader_expect(&pi->rd, "ll")) {
+		    add_value(pi, Qnil);
+		} else {
+		    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected null");
+		    return;
+		}
+	    } else if ('a' == c) {
+		struct _NumInfo	ni;
+
+		c = reader_get(&pi->rd);
+		if ('N' != c && 'n' != c) {
+		    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected NaN");
+		    return;
+		}
+		ni.str = pi->rd.str;
+		ni.i = 0;
+		ni.num = 0;
+		ni.div = 1;
+		ni.len = 0;
+		ni.exp = 0;
+		ni.dec_cnt = 0;
+		ni.big = 0;
+		ni.infinity = 0;
+		ni.nan = 1;
+		ni.neg = 0;
+		ni.no_big = (FloatDec == pi->options.bigdec_load);
+		add_num_value(pi, &ni);
+	    } else {
+		oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "invalid token");
+		return;
+	    }
+	    break;
+	case '/':
+	    skip_comment(pi);
+	    break;
+	case '\0':
+	    return;
+	default:
+	    oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "unexpected character");
+	    return;
+	}
+	if (err_has(&pi->err)) {
+	    return;
+	}
+	if (Qundef != pi->proc && stack_empty(&pi->stack)) {
+	    if (Qnil == pi->proc) {
+		rb_yield(stack_head_val(&pi->stack));
+	    } else {
+#if HAS_PROC_WITH_BLOCK
+		VALUE	args[1];
+
+		*args = stack_head_val(&pi->stack);
+		rb_proc_call_with_block(pi->proc, 1, args, Qnil);
+#else
+		oj_set_error_at(pi, rb_eNotImpError, __FILE__, __LINE__,
+				 "Calling a Proc with a block not supported in this version. Use func() {|x| } syntax instead.");
+		return;
+#endif
+	    }
+	}
+    }
+}
+
+static VALUE
+protect_parse(VALUE pip) {
+    oj_sparse2((ParseInfo)pip);
+
+    return Qnil;
+}
+
+VALUE
+oj_pi_sparse(int argc, VALUE *argv, ParseInfo pi, int fd) {
+    volatile VALUE	input;
+    volatile VALUE	wrapped_stack;
+    VALUE		result = Qnil;
+    int			line = 0;
+
+    if (argc < 1) {
+	rb_raise(rb_eArgError, "Wrong number of arguments to parse.");
+    }
+    input = argv[0];
+    if (2 == argc) {
+	oj_parse_options(argv[1], &pi->options);
+    }
+    if (Qnil == input && Yes == pi->options.nilnil) {
+	return Qnil;
+    }
+    if (rb_block_given_p()) {
+	pi->proc = Qnil;
+    } else {
+	pi->proc = Qundef;
+    }
+    pi->cbc = (void*)0;
+
+    oj_reader_init(&pi->rd, input, fd);
+    pi->json = 0; // indicates reader is in use
+
+    if (Yes == pi->options.circular) {
+	pi->circ_array = oj_circ_array_new();
+    } else {
+	pi->circ_array = 0;
+    }
+    if (No == pi->options.allow_gc) {
+	rb_gc_disable();
+    }
+    // GC can run at any time. When it runs any Object created by C will be
+    // freed. We protect against this by wrapping the value stack in a ruby
+    // data object and poviding a mark function for ruby objects on the
+    // value stack (while it is in scope).
+    wrapped_stack = oj_stack_init(&pi->stack);
+    rb_protect(protect_parse, (VALUE)pi, &line);
+    result = stack_head_val(&pi->stack);
+    DATA_PTR(wrapped_stack) = 0;
+    if (No == pi->options.allow_gc) {
+	rb_gc_enable();
+    }
+    if (!err_has(&pi->err)) {
+	// If the stack is not empty then the JSON terminated early.
+	Val	v;
+
+	if (0 != (v = stack_peek(&pi->stack))) {
+	    switch (v->next) {
+	    case NEXT_ARRAY_NEW:
+	    case NEXT_ARRAY_ELEMENT:
+	    case NEXT_ARRAY_COMMA:
+		oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "Array not terminated");
+		break;
+	    case NEXT_HASH_NEW:
+	    case NEXT_HASH_KEY:
+	    case NEXT_HASH_COLON:
+	    case NEXT_HASH_VALUE:
+	    case NEXT_HASH_COMMA:
+		oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "Hash/Object not terminated");
+		break;
+	    default:
+		oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not terminated");
+	    }
+	}
+    }
+    // proceed with cleanup
+    if (0 != pi->circ_array) {
+	oj_circ_array_free(pi->circ_array);
+    }
+    stack_cleanup(&pi->stack);
+    if (0 != fd) {
+	close(fd);
+    }
+    if (0 != line) {
+	rb_jump_tag(line);
+    }
+    if (err_has(&pi->err)) {
+	oj_err_raise(&pi->err);
+    }
+    return result;
+}