ffi-yajl 0.0.1-universal-java → 0.0.2-universal-java

data/ext/libyajl2/vendored/src/yajl_encode.h

@@ -1,34 +0,0 @@
-/*
- * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
-#ifndef __YAJL_ENCODE_H__
-#define __YAJL_ENCODE_H__
-
-#include "yajl_buf.h"
-#include "api/yajl_gen.h"
-
-void yajl_string_encode(const yajl_print_t printer,
-                        void * ctx,
-                        const unsigned char * str,
-                        size_t length,
-                        int escape_solidus);
-
-void yajl_string_decode(yajl_buf buf, const unsigned char * str,
-                        size_t length);
-
-int yajl_string_validate_utf8(const unsigned char * s, size_t len);
-
-#endif

data/ext/libyajl2/vendored/src/yajl_gen.c

@@ -1,354 +0,0 @@
-/*
- * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
-#include "api/yajl_gen.h"
-#include "yajl_buf.h"
-#include "yajl_encode.h"
-
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <math.h>
-#include <stdarg.h>
-
-typedef enum {
-    yajl_gen_start,
-    yajl_gen_map_start,
-    yajl_gen_map_key,
-    yajl_gen_map_val,
-    yajl_gen_array_start,
-    yajl_gen_in_array,
-    yajl_gen_complete,
-    yajl_gen_error
-} yajl_gen_state;
-
-struct yajl_gen_t
-{
-    unsigned int flags;
-    unsigned int depth;
-    const char * indentString;
-    yajl_gen_state state[YAJL_MAX_DEPTH];
-    yajl_print_t print;
-    void * ctx; /* yajl_buf */
-    /* memory allocation routines */
-    yajl_alloc_funcs alloc;
-};
-
-int
-yajl_gen_config(yajl_gen g, yajl_gen_option opt, ...)
-{
-    int rv = 1;
-    va_list ap;
-    va_start(ap, opt);
-
-    switch(opt) {
-        case yajl_gen_beautify:
-        case yajl_gen_validate_utf8:
-        case yajl_gen_escape_solidus:
-            if (va_arg(ap, int)) g->flags |= opt;
-            else g->flags &= ~opt;
-            break;
-        case yajl_gen_indent_string: {
-            const char *indent = va_arg(ap, const char *);
-            g->indentString = indent;
-            for (; *indent; indent++) {
-                if (*indent != '\n'
-                    && *indent != '\v'
-                    && *indent != '\f'
-                    && *indent != '\t'
-                    && *indent != '\r'
-                    && *indent != ' ')
-                {
-                    g->indentString = NULL;
-                    rv = 0;
-                }
-            }
-            break;
-        }
-        case yajl_gen_print_callback:
-            yajl_buf_free(g->ctx);
-            g->print = va_arg(ap, const yajl_print_t);
-            g->ctx = va_arg(ap, void *);
-            break;
-        default:
-            rv = 0;
-    }
-
-    va_end(ap);
-
-    return rv;
-}
-
-
-
-yajl_gen
-yajl_gen_alloc(const yajl_alloc_funcs * afs)
-{
-    yajl_gen g = NULL;
-    yajl_alloc_funcs afsBuffer;
-
-    /* first order of business is to set up memory allocation routines */
-    if (afs != NULL) {
-        if (afs->malloc == NULL || afs->realloc == NULL || afs->free == NULL)
-        {
-            return NULL;
-        }
-    } else {
-        yajl_set_default_alloc_funcs(&afsBuffer);
-        afs = &afsBuffer;
-    }
-
-    g = (yajl_gen) YA_MALLOC(afs, sizeof(struct yajl_gen_t));
-    if (!g) return NULL;
-
-    memset((void *) g, 0, sizeof(struct yajl_gen_t));
-    /* copy in pointers to allocation routines */
-    memcpy((void *) &(g->alloc), (void *) afs, sizeof(yajl_alloc_funcs));
-
-    g->print = (yajl_print_t)&yajl_buf_append;
-    g->ctx = yajl_buf_alloc(&(g->alloc));
-    g->indentString = "    ";
-
-    return g;
-}
-
-void
-yajl_gen_free(yajl_gen g)
-{
-    if (g->print == (yajl_print_t)&yajl_buf_append) yajl_buf_free((yajl_buf)g->ctx);
-    YA_FREE(&(g->alloc), g);
-}
-
-#define INSERT_SEP \
-    if (g->state[g->depth] == yajl_gen_map_key ||               \
-        g->state[g->depth] == yajl_gen_in_array) {              \
-        g->print(g->ctx, ",", 1);                               \
-        if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);               \
-    } else if (g->state[g->depth] == yajl_gen_map_val) {        \
-        g->print(g->ctx, ":", 1);                               \
-        if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, " ", 1);                \
-   } 
-
-#define INSERT_WHITESPACE                                               \
-    if ((g->flags & yajl_gen_beautify)) {                                                    \
-        if (g->state[g->depth] != yajl_gen_map_val) {                   \
-            unsigned int _i;                                            \
-            for (_i=0;_i<g->depth;_i++)                                 \
-                g->print(g->ctx,                                        \
-                         g->indentString,                               \
-                         (unsigned int)strlen(g->indentString));        \
-        }                                                               \
-    }
-
-#define ENSURE_NOT_KEY \
-    if (g->state[g->depth] == yajl_gen_map_key ||       \
-        g->state[g->depth] == yajl_gen_map_start)  {    \
-        return yajl_gen_keys_must_be_strings;           \
-    }                                                   \
-
-/* check that we're not complete, or in error state.  in a valid state
- * to be generating */
-#define ENSURE_VALID_STATE \
-    if (g->state[g->depth] == yajl_gen_error) {   \
-        return yajl_gen_in_error_state;\
-    } else if (g->state[g->depth] == yajl_gen_complete) {   \
-        return yajl_gen_generation_complete;                \
-    }
-
-#define INCREMENT_DEPTH \
-    if (++(g->depth) >= YAJL_MAX_DEPTH) return yajl_max_depth_exceeded;
-
-#define DECREMENT_DEPTH \
-  if (--(g->depth) >= YAJL_MAX_DEPTH) return yajl_gen_error;
-
-#define APPENDED_ATOM \
-    switch (g->state[g->depth]) {                   \
-        case yajl_gen_start:                        \
-            g->state[g->depth] = yajl_gen_complete; \
-            break;                                  \
-        case yajl_gen_map_start:                    \
-        case yajl_gen_map_key:                      \
-            g->state[g->depth] = yajl_gen_map_val;  \
-            break;                                  \
-        case yajl_gen_array_start:                  \
-            g->state[g->depth] = yajl_gen_in_array; \
-            break;                                  \
-        case yajl_gen_map_val:                      \
-            g->state[g->depth] = yajl_gen_map_key;  \
-            break;                                  \
-        default:                                    \
-            break;                                  \
-    }                                               \
-
-#define FINAL_NEWLINE                                        \
-    if ((g->flags & yajl_gen_beautify) && g->state[g->depth] == yajl_gen_complete) \
-        g->print(g->ctx, "\n", 1);
-
-yajl_gen_status
-yajl_gen_integer(yajl_gen g, long long int number)
-{
-    char i[32];
-    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
-    sprintf(i, "%lld", number);
-    g->print(g->ctx, i, (unsigned int)strlen(i));
-    APPENDED_ATOM;
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-#if defined(_WIN32) || defined(WIN32)
-#include <float.h>
-#define isnan _isnan
-#define isinf !_finite
-#endif
-
-yajl_gen_status
-yajl_gen_double(yajl_gen g, double number)
-{
-    char i[32];
-    ENSURE_VALID_STATE; ENSURE_NOT_KEY; 
-    if (isnan(number) || isinf(number)) return yajl_gen_invalid_number;
-    INSERT_SEP; INSERT_WHITESPACE;
-    sprintf(i, "%.20g", number);
-    if (strspn(i, "0123456789-") == strlen(i)) {
-        strcat(i, ".0");
-    }
-    g->print(g->ctx, i, (unsigned int)strlen(i));
-    APPENDED_ATOM;
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-yajl_gen_status
-yajl_gen_number(yajl_gen g, const char * s, size_t l)
-{
-    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
-    g->print(g->ctx, s, l);
-    APPENDED_ATOM;
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-yajl_gen_status
-yajl_gen_string(yajl_gen g, const unsigned char * str,
-                size_t len)
-{
-    // if validation is enabled, check that the string is valid utf8
-    // XXX: This checking could be done a little faster, in the same pass as
-    // the string encoding
-    if (g->flags & yajl_gen_validate_utf8) {
-        if (!yajl_string_validate_utf8(str, len)) {
-            return yajl_gen_invalid_string;
-        }
-    }
-    ENSURE_VALID_STATE; INSERT_SEP; INSERT_WHITESPACE;
-    g->print(g->ctx, "\"", 1);
-    yajl_string_encode(g->print, g->ctx, str, len, g->flags & yajl_gen_escape_solidus);
-    g->print(g->ctx, "\"", 1);
-    APPENDED_ATOM;
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-yajl_gen_status
-yajl_gen_null(yajl_gen g)
-{
-    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
-    g->print(g->ctx, "null", strlen("null"));
-    APPENDED_ATOM;
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-yajl_gen_status
-yajl_gen_bool(yajl_gen g, int boolean)
-{
-    const char * val = boolean ? "true" : "false";
-
-	ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
-    g->print(g->ctx, val, (unsigned int)strlen(val));
-    APPENDED_ATOM;
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-yajl_gen_status
-yajl_gen_map_open(yajl_gen g)
-{
-    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
-    INCREMENT_DEPTH; 
-    
-    g->state[g->depth] = yajl_gen_map_start;
-    g->print(g->ctx, "{", 1);
-    if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-yajl_gen_status
-yajl_gen_map_close(yajl_gen g)
-{
-    ENSURE_VALID_STATE; 
-    DECREMENT_DEPTH;
-    
-    if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);
-    APPENDED_ATOM;
-    INSERT_WHITESPACE;
-    g->print(g->ctx, "}", 1);
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-yajl_gen_status
-yajl_gen_array_open(yajl_gen g)
-{
-    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
-    INCREMENT_DEPTH; 
-    g->state[g->depth] = yajl_gen_array_start;
-    g->print(g->ctx, "[", 1);
-    if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-yajl_gen_status
-yajl_gen_array_close(yajl_gen g)
-{
-    ENSURE_VALID_STATE;
-    DECREMENT_DEPTH;
-    if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);
-    APPENDED_ATOM;
-    INSERT_WHITESPACE;
-    g->print(g->ctx, "]", 1);
-    FINAL_NEWLINE;
-    return yajl_gen_status_ok;
-}
-
-yajl_gen_status
-yajl_gen_get_buf(yajl_gen g, const unsigned char ** buf,
-                 size_t * len)
-{
-    if (g->print != (yajl_print_t)&yajl_buf_append) return yajl_gen_no_buf;
-    *buf = yajl_buf_data((yajl_buf)g->ctx);
-    *len = yajl_buf_len((yajl_buf)g->ctx);
-    return yajl_gen_status_ok;
-}
-
-void
-yajl_gen_clear(yajl_gen g)
-{
-    if (g->print == (yajl_print_t)&yajl_buf_append) yajl_buf_clear((yajl_buf)g->ctx);
-}

data/ext/libyajl2/vendored/src/yajl_lex.c

@@ -1,763 +0,0 @@
-/*
- * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
-#include "yajl_lex.h"
-#include "yajl_buf.h"
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <assert.h>
-#include <string.h>
-
-#ifdef YAJL_LEXER_DEBUG
-static const char *
-tokToStr(yajl_tok tok) 
-{
-    switch (tok) {
-        case yajl_tok_bool: return "bool";
-        case yajl_tok_colon: return "colon";
-        case yajl_tok_comma: return "comma";
-        case yajl_tok_eof: return "eof";
-        case yajl_tok_error: return "error";
-        case yajl_tok_left_brace: return "brace";
-        case yajl_tok_left_bracket: return "bracket";
-        case yajl_tok_null: return "null";
-        case yajl_tok_integer: return "integer";
-        case yajl_tok_double: return "double";
-        case yajl_tok_right_brace: return "brace";
-        case yajl_tok_right_bracket: return "bracket";
-        case yajl_tok_string: return "string";
-        case yajl_tok_string_with_escapes: return "string_with_escapes";
-    }
-    return "unknown";
-}
-#endif
-
-/* Impact of the stream parsing feature on the lexer:
- *
- * YAJL support stream parsing.  That is, the ability to parse the first
- * bits of a chunk of JSON before the last bits are available (still on
- * the network or disk).  This makes the lexer more complex.  The
- * responsibility of the lexer is to handle transparently the case where
- * a chunk boundary falls in the middle of a token.  This is
- * accomplished is via a buffer and a character reading abstraction. 
- *
- * Overview of implementation
- *
- * When we lex to end of input string before end of token is hit, we
- * copy all of the input text composing the token into our lexBuf.
- * 
- * Every time we read a character, we do so through the readChar function.
- * readChar's responsibility is to handle pulling all chars from the buffer
- * before pulling chars from input text
- */
-
-struct yajl_lexer_t {
-    /* the overal line and char offset into the data */
-    size_t lineOff;
-    size_t charOff;
-
-    /* error */
-    yajl_lex_error error;
-
-    /* a input buffer to handle the case where a token is spread over
-     * multiple chunks */ 
-    yajl_buf buf;
-
-    /* in the case where we have data in the lexBuf, bufOff holds
-     * the current offset into the lexBuf. */
-    size_t bufOff;
-
-    /* are we using the lex buf? */
-    unsigned int bufInUse;
-
-    /* shall we allow comments? */
-    unsigned int allowComments;
-
-    /* shall we validate utf8 inside strings? */
-    unsigned int validateUTF8;
-
-    yajl_alloc_funcs * alloc;
-};
-
-#define readChar(lxr, txt, off)                      \
-    (((lxr)->bufInUse && yajl_buf_len((lxr)->buf) && lxr->bufOff < yajl_buf_len((lxr)->buf)) ? \
-     (*((const unsigned char *) yajl_buf_data((lxr)->buf) + ((lxr)->bufOff)++)) : \
-     ((txt)[(*(off))++]))
-
-#define unreadChar(lxr, off) ((*(off) > 0) ? (*(off))-- : ((lxr)->bufOff--))
-
-yajl_lexer
-yajl_lex_alloc(yajl_alloc_funcs * alloc,
-               unsigned int allowComments, unsigned int validateUTF8)
-{
-    yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t));
-    memset((void *) lxr, 0, sizeof(struct yajl_lexer_t));
-    lxr->buf = yajl_buf_alloc(alloc);
-    lxr->allowComments = allowComments;
-    lxr->validateUTF8 = validateUTF8;
-    lxr->alloc = alloc;
-    return lxr;
-}
-
-void
-yajl_lex_free(yajl_lexer lxr)
-{
-    yajl_buf_free(lxr->buf);
-    YA_FREE(lxr->alloc, lxr);
-    return;
-}
-
-/* a lookup table which lets us quickly determine three things:
- * VEC - valid escaped control char
- * note.  the solidus '/' may be escaped or not.
- * IJC - invalid json char
- * VHC - valid hex char
- * NFP - needs further processing (from a string scanning perspective)
- * NUC - needs utf8 checking when enabled (from a string scanning perspective)
- */
-#define VEC 0x01
-#define IJC 0x02
-#define VHC 0x04
-#define NFP 0x08
-#define NUC 0x10
-
-static const char charLookupTable[256] =
-{
-/*00*/ IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    ,
-/*08*/ IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    ,
-/*10*/ IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    ,
-/*18*/ IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    ,
-
-/*20*/ 0      , 0      , NFP|VEC|IJC, 0      , 0      , 0      , 0      , 0      ,
-/*28*/ 0      , 0      , 0      , 0      , 0      , 0      , 0      , VEC    ,
-/*30*/ VHC    , VHC    , VHC    , VHC    , VHC    , VHC    , VHC    , VHC    ,
-/*38*/ VHC    , VHC    , 0      , 0      , 0      , 0      , 0      , 0      ,
-
-/*40*/ 0      , VHC    , VHC    , VHC    , VHC    , VHC    , VHC    , 0      ,
-/*48*/ 0      , 0      , 0      , 0      , 0      , 0      , 0      , 0      ,
-/*50*/ 0      , 0      , 0      , 0      , 0      , 0      , 0      , 0      ,
-/*58*/ 0      , 0      , 0      , 0      , NFP|VEC|IJC, 0      , 0      , 0      ,
-
-/*60*/ 0      , VHC    , VEC|VHC, VHC    , VHC    , VHC    , VEC|VHC, 0      ,
-/*68*/ 0      , 0      , 0      , 0      , 0      , 0      , VEC    , 0      ,
-/*70*/ 0      , 0      , VEC    , 0      , VEC    , 0      , 0      , 0      ,
-/*78*/ 0      , 0      , 0      , 0      , 0      , 0      , 0      , 0      ,
-
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
-       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC
-};
-
-/** process a variable length utf8 encoded codepoint.
- *
- *  returns:
- *    yajl_tok_string - if valid utf8 char was parsed and offset was
- *                      advanced
- *    yajl_tok_eof - if end of input was hit before validation could
- *                   complete
- *    yajl_tok_error - if invalid utf8 was encountered
- * 
- *  NOTE: on error the offset will point to the first char of the
- *  invalid utf8 */
-#define UTF8_CHECK_EOF if (*offset >= jsonTextLen) { return yajl_tok_eof; }
-
-static yajl_tok
-yajl_lex_utf8_char(yajl_lexer lexer, const unsigned char * jsonText,
-                   size_t jsonTextLen, size_t * offset,
-                   unsigned char curChar)
-{
-    if (curChar <= 0x7f) {
-        /* single byte */
-        return yajl_tok_string;
-    } else if ((curChar >> 5) == 0x6) {
-        /* two byte */ 
-        UTF8_CHECK_EOF;
-        curChar = readChar(lexer, jsonText, offset);
-        if ((curChar >> 6) == 0x2) return yajl_tok_string;
-    } else if ((curChar >> 4) == 0x0e) {
-        /* three byte */
-        UTF8_CHECK_EOF;
-        curChar = readChar(lexer, jsonText, offset);
-        if ((curChar >> 6) == 0x2) {
-            UTF8_CHECK_EOF;
-            curChar = readChar(lexer, jsonText, offset);
-            if ((curChar >> 6) == 0x2) return yajl_tok_string;
-        }
-    } else if ((curChar >> 3) == 0x1e) {
-        /* four byte */
-        UTF8_CHECK_EOF;
-        curChar = readChar(lexer, jsonText, offset);
-        if ((curChar >> 6) == 0x2) {
-            UTF8_CHECK_EOF;
-            curChar = readChar(lexer, jsonText, offset);
-            if ((curChar >> 6) == 0x2) {
-                UTF8_CHECK_EOF;
-                curChar = readChar(lexer, jsonText, offset);
-                if ((curChar >> 6) == 0x2) return yajl_tok_string;
-            }
-        }
-    } 
-
-    return yajl_tok_error;
-}
-
-/* lex a string.  input is the lexer, pointer to beginning of
- * json text, and start of string (offset).
- * a token is returned which has the following meanings:
- * yajl_tok_string: lex of string was successful.  offset points to
- *                  terminating '"'.
- * yajl_tok_eof: end of text was encountered before we could complete
- *               the lex.
- * yajl_tok_error: embedded in the string were unallowable chars.  offset
- *               points to the offending char
- */
-#define STR_CHECK_EOF \
-if (*offset >= jsonTextLen) { \
-   tok = yajl_tok_eof; \
-   goto finish_string_lex; \
-}
-
-/** scan a string for interesting characters that might need further
- *  review.  return the number of chars that are uninteresting and can
- *  be skipped.
- * (lth) hi world, any thoughts on how to make this routine faster? */
-static size_t
-yajl_string_scan(const unsigned char * buf, size_t len, int utf8check)
-{
-    unsigned char mask = IJC|NFP|(utf8check ? NUC : 0);
-    size_t skip = 0;
-    while (skip < len && !(charLookupTable[*buf] & mask))
-    {
-        skip++;
-        buf++;
-    }
-    return skip;
-}
-
-static yajl_tok
-yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText,
-                size_t jsonTextLen, size_t * offset)
-{
-    yajl_tok tok = yajl_tok_error;
-    int hasEscapes = 0;
-
-    for (;;) {
-        unsigned char curChar;
-
-        /* now jump into a faster scanning routine to skip as much
-         * of the buffers as possible */
-        {
-            const unsigned char * p;
-            size_t len;
-            
-            if ((lexer->bufInUse && yajl_buf_len(lexer->buf) &&
-                 lexer->bufOff < yajl_buf_len(lexer->buf)))
-            {
-                p = ((const unsigned char *) yajl_buf_data(lexer->buf) +
-                     (lexer->bufOff));
-                len = yajl_buf_len(lexer->buf) - lexer->bufOff;
-                lexer->bufOff += yajl_string_scan(p, len, lexer->validateUTF8);
-            }                
-            else if (*offset < jsonTextLen) 
-            {
-                p = jsonText + *offset;
-                len = jsonTextLen - *offset;
-                *offset += yajl_string_scan(p, len, lexer->validateUTF8);
-            }
-        }
-
-        STR_CHECK_EOF;
-
-        curChar = readChar(lexer, jsonText, offset);
-
-        /* quote terminates */
-        if (curChar == '"') {
-            tok = yajl_tok_string;
-            break;
-        }
-        /* backslash escapes a set of control chars, */
-        else if (curChar == '\\') {
-            hasEscapes = 1;
-            STR_CHECK_EOF;
-
-            /* special case \u */
-            curChar = readChar(lexer, jsonText, offset);
-            if (curChar == 'u') {
-                unsigned int i = 0;
-
-                for (i=0;i<4;i++) {
-                    STR_CHECK_EOF;                
-                    curChar = readChar(lexer, jsonText, offset);                
-                    if (!(charLookupTable[curChar] & VHC)) {
-                        /* back up to offending char */
-                        unreadChar(lexer, offset);
-                        lexer->error = yajl_lex_string_invalid_hex_char;
-                        goto finish_string_lex;
-                    }
-                }
-            } else if (!(charLookupTable[curChar] & VEC)) {
-                /* back up to offending char */
-                unreadChar(lexer, offset);
-                lexer->error = yajl_lex_string_invalid_escaped_char;
-                goto finish_string_lex;                
-            } 
-        }
-        /* when not validating UTF8 it's a simple table lookup to determine
-         * if the present character is invalid */
-        else if(charLookupTable[curChar] & IJC) {
-            /* back up to offending char */
-            unreadChar(lexer, offset);
-            lexer->error = yajl_lex_string_invalid_json_char;
-            goto finish_string_lex;                
-        }
-        /* when in validate UTF8 mode we need to do some extra work */
-        else if (lexer->validateUTF8) {
-            yajl_tok t = yajl_lex_utf8_char(lexer, jsonText, jsonTextLen,
-                                            offset, curChar);
-            
-            if (t == yajl_tok_eof) {
-                tok = yajl_tok_eof;
-                goto finish_string_lex;
-            } else if (t == yajl_tok_error) {
-                lexer->error = yajl_lex_string_invalid_utf8;
-                goto finish_string_lex;
-            } 
-        }
-        /* accept it, and move on */ 
-    }
-  finish_string_lex:
-    /* tell our buddy, the parser, wether he needs to process this string
-     * again */
-    if (hasEscapes && tok == yajl_tok_string) {
-        tok = yajl_tok_string_with_escapes;
-    } 
-
-    return tok;
-}
-
-#define RETURN_IF_EOF if (*offset >= jsonTextLen) return yajl_tok_eof;
-
-static yajl_tok
-yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText,
-                size_t jsonTextLen, size_t * offset)
-{
-    /** XXX: numbers are the only entities in json that we must lex
-     *       _beyond_ in order to know that they are complete.  There
-     *       is an ambiguous case for integers at EOF. */
-
-    unsigned char c;
-
-    yajl_tok tok = yajl_tok_integer;
-
-    RETURN_IF_EOF;    
-    c = readChar(lexer, jsonText, offset);
-
-    /* optional leading minus */
-    if (c == '-') {
-        RETURN_IF_EOF;    
-        c = readChar(lexer, jsonText, offset); 
-    }
-
-    /* a single zero, or a series of integers */
-    if (c == '0') {
-        RETURN_IF_EOF;    
-        c = readChar(lexer, jsonText, offset); 
-    } else if (c >= '1' && c <= '9') {
-        do {
-            RETURN_IF_EOF;    
-            c = readChar(lexer, jsonText, offset); 
-        } while (c >= '0' && c <= '9');
-    } else {
-        unreadChar(lexer, offset);
-        lexer->error = yajl_lex_missing_integer_after_minus;
-        return yajl_tok_error;
-    }
-
-    /* optional fraction (indicates this is floating point) */
-    if (c == '.') {
-        int numRd = 0;
-        
-        RETURN_IF_EOF;
-        c = readChar(lexer, jsonText, offset); 
-
-        while (c >= '0' && c <= '9') {
-            numRd++;
-            RETURN_IF_EOF;
-            c = readChar(lexer, jsonText, offset); 
-        } 
-
-        if (!numRd) {
-            unreadChar(lexer, offset);
-            lexer->error = yajl_lex_missing_integer_after_decimal;
-            return yajl_tok_error;
-        }
-        tok = yajl_tok_double;
-    }
-
-    /* optional exponent (indicates this is floating point) */
-    if (c == 'e' || c == 'E') {
-        RETURN_IF_EOF;
-        c = readChar(lexer, jsonText, offset); 
-
-        /* optional sign */
-        if (c == '+' || c == '-') {
-            RETURN_IF_EOF;
-            c = readChar(lexer, jsonText, offset); 
-        }
-
-        if (c >= '0' && c <= '9') {
-            do {
-                RETURN_IF_EOF;
-                c = readChar(lexer, jsonText, offset); 
-            } while (c >= '0' && c <= '9');
-        } else {
-            unreadChar(lexer, offset);
-            lexer->error = yajl_lex_missing_integer_after_exponent;
-            return yajl_tok_error;
-        }
-        tok = yajl_tok_double;
-    }
-    
-    /* we always go "one too far" */
-    unreadChar(lexer, offset);
-    
-    return tok;
-}
-
-static yajl_tok
-yajl_lex_comment(yajl_lexer lexer, const unsigned char * jsonText,
-                 size_t jsonTextLen, size_t * offset)
-{
-    unsigned char c;
-
-    yajl_tok tok = yajl_tok_comment;
-
-    RETURN_IF_EOF;    
-    c = readChar(lexer, jsonText, offset);
-
-    /* either slash or star expected */
-    if (c == '/') {
-        /* now we throw away until end of line */
-        do {
-            RETURN_IF_EOF;    
-            c = readChar(lexer, jsonText, offset); 
-        } while (c != '\n');
-    } else if (c == '*') {
-        /* now we throw away until end of comment */        
-        for (;;) {
-            RETURN_IF_EOF;    
-            c = readChar(lexer, jsonText, offset); 
-            if (c == '*') {
-                RETURN_IF_EOF;    
-                c = readChar(lexer, jsonText, offset);                 
-                if (c == '/') {
-                    break;
-                } else {
-                    unreadChar(lexer, offset);
-                }
-            }
-        }
-    } else {
-        lexer->error = yajl_lex_invalid_char;
-        tok = yajl_tok_error;
-    }
-    
-    return tok;
-}
-
-yajl_tok
-yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText,
-             size_t jsonTextLen, size_t * offset,
-             const unsigned char ** outBuf, size_t * outLen)
-{
-    yajl_tok tok = yajl_tok_error;
-    unsigned char c;
-    size_t startOffset = *offset;
-
-    *outBuf = NULL;
-    *outLen = 0;
-
-    for (;;) {
-        assert(*offset <= jsonTextLen);
-
-        if (*offset >= jsonTextLen) {
-            tok = yajl_tok_eof;
-            goto lexed;
-        }
-
-        c = readChar(lexer, jsonText, offset);
-
-        switch (c) {
-            case '{':
-                tok = yajl_tok_left_bracket;
-                goto lexed;
-            case '}':
-                tok = yajl_tok_right_bracket;
-                goto lexed;
-            case '[':
-                tok = yajl_tok_left_brace;
-                goto lexed;
-            case ']':
-                tok = yajl_tok_right_brace;
-                goto lexed;
-            case ',':
-                tok = yajl_tok_comma;
-                goto lexed;
-            case ':':
-                tok = yajl_tok_colon;
-                goto lexed;
-            case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
-                startOffset++;
-                break;
-            case 't': {
-                const char * want = "rue";
-                do {
-                    if (*offset >= jsonTextLen) {
-                        tok = yajl_tok_eof;
-                        goto lexed;
-                    }
-                    c = readChar(lexer, jsonText, offset);
-                    if (c != *want) {
-                        unreadChar(lexer, offset);
-                        lexer->error = yajl_lex_invalid_string;
-                        tok = yajl_tok_error;
-                        goto lexed;
-                    }
-                } while (*(++want));
-                tok = yajl_tok_bool;
-                goto lexed;
-            }
-            case 'f': {
-                const char * want = "alse";
-                do {
-                    if (*offset >= jsonTextLen) {
-                        tok = yajl_tok_eof;
-                        goto lexed;
-                    }
-                    c = readChar(lexer, jsonText, offset);
-                    if (c != *want) {
-                        unreadChar(lexer, offset);
-                        lexer->error = yajl_lex_invalid_string;
-                        tok = yajl_tok_error;
-                        goto lexed;
-                    }
-                } while (*(++want));
-                tok = yajl_tok_bool;
-                goto lexed;
-            }
-            case 'n': {
-                const char * want = "ull";
-                do {
-                    if (*offset >= jsonTextLen) {
-                        tok = yajl_tok_eof;
-                        goto lexed;
-                    }
-                    c = readChar(lexer, jsonText, offset);
-                    if (c != *want) {
-                        unreadChar(lexer, offset);
-                        lexer->error = yajl_lex_invalid_string;
-                        tok = yajl_tok_error;
-                        goto lexed;
-                    }
-                } while (*(++want));
-                tok = yajl_tok_null;
-                goto lexed;
-            }
-            case '"': {
-                tok = yajl_lex_string(lexer, (const unsigned char *) jsonText,
-                                      jsonTextLen, offset);
-                goto lexed;
-            }
-            case '-':
-            case '0': case '1': case '2': case '3': case '4': 
-            case '5': case '6': case '7': case '8': case '9': {
-                /* integer parsing wants to start from the beginning */
-                unreadChar(lexer, offset);
-                tok = yajl_lex_number(lexer, (const unsigned char *) jsonText,
-                                      jsonTextLen, offset);
-                goto lexed;
-            }
-            case '/':
-                /* hey, look, a probable comment!  If comments are disabled
-                 * it's an error. */
-                if (!lexer->allowComments) {
-                    unreadChar(lexer, offset);
-                    lexer->error = yajl_lex_unallowed_comment;
-                    tok = yajl_tok_error;
-                    goto lexed;
-                }
-                /* if comments are enabled, then we should try to lex
-                 * the thing.  possible outcomes are
-                 * - successful lex (tok_comment, which means continue),
-                 * - malformed comment opening (slash not followed by
-                 *   '*' or '/') (tok_error)
-                 * - eof hit. (tok_eof) */
-                tok = yajl_lex_comment(lexer, (const unsigned char *) jsonText,
-                                       jsonTextLen, offset);
-                if (tok == yajl_tok_comment) {
-                    /* "error" is silly, but that's the initial
-                     * state of tok.  guilty until proven innocent. */  
-                    tok = yajl_tok_error;
-                    yajl_buf_clear(lexer->buf);
-                    lexer->bufInUse = 0;
-                    startOffset = *offset; 
-                    break;
-                }
-                /* hit error or eof, bail */
-                goto lexed;
-            default:
-                lexer->error = yajl_lex_invalid_char;
-                tok = yajl_tok_error;
-                goto lexed;
-        }
-    }
-
-
-  lexed:
-    /* need to append to buffer if the buffer is in use or
-     * if it's an EOF token */
-    if (tok == yajl_tok_eof || lexer->bufInUse) {
-        if (!lexer->bufInUse) yajl_buf_clear(lexer->buf);
-        lexer->bufInUse = 1;
-        yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset);
-        lexer->bufOff = 0;
-        
-        if (tok != yajl_tok_eof) {
-            *outBuf = yajl_buf_data(lexer->buf);
-            *outLen = yajl_buf_len(lexer->buf);
-            lexer->bufInUse = 0;
-        }
-    } else if (tok != yajl_tok_error) {
-        *outBuf = jsonText + startOffset;
-        *outLen = *offset - startOffset;
-    }
-
-    /* special case for strings. skip the quotes. */
-    if (tok == yajl_tok_string || tok == yajl_tok_string_with_escapes)
-    {
-        assert(*outLen >= 2);
-        (*outBuf)++;
-        *outLen -= 2; 
-    }
-
-
-#ifdef YAJL_LEXER_DEBUG
-    if (tok == yajl_tok_error) {
-        printf("lexical error: %s\n",
-               yajl_lex_error_to_string(yajl_lex_get_error(lexer)));
-    } else if (tok == yajl_tok_eof) {
-        printf("EOF hit\n");
-    } else {
-        printf("lexed %s: '", tokToStr(tok));
-        fwrite(*outBuf, 1, *outLen, stdout);
-        printf("'\n");
-    }
-#endif
-
-    return tok;
-}
-
-const char *
-yajl_lex_error_to_string(yajl_lex_error error)
-{
-    switch (error) {
-        case yajl_lex_e_ok:
-            return "ok, no error";
-        case yajl_lex_string_invalid_utf8:
-            return "invalid bytes in UTF8 string.";
-        case yajl_lex_string_invalid_escaped_char:
-            return "inside a string, '\\' occurs before a character "
-                   "which it may not.";
-        case yajl_lex_string_invalid_json_char:            
-            return "invalid character inside string.";
-        case yajl_lex_string_invalid_hex_char:
-            return "invalid (non-hex) character occurs after '\\u' inside "
-                   "string.";
-        case yajl_lex_invalid_char:
-            return "invalid char in json text.";
-        case yajl_lex_invalid_string:
-            return "invalid string in json text.";
-        case yajl_lex_missing_integer_after_exponent:
-            return "malformed number, a digit is required after the exponent.";
-        case yajl_lex_missing_integer_after_decimal:
-            return "malformed number, a digit is required after the "
-                   "decimal point.";
-        case yajl_lex_missing_integer_after_minus:
-            return "malformed number, a digit is required after the "
-                   "minus sign.";
-        case yajl_lex_unallowed_comment:
-            return "probable comment found in input text, comments are "
-                   "not enabled.";
-    }
-    return "unknown error code";
-}
-
-
-/** allows access to more specific information about the lexical
- *  error when yajl_lex_lex returns yajl_tok_error. */
-yajl_lex_error
-yajl_lex_get_error(yajl_lexer lexer)
-{
-    if (lexer == NULL) return (yajl_lex_error) -1;
-    return lexer->error;
-}
-
-size_t yajl_lex_current_line(yajl_lexer lexer)
-{
-    return lexer->lineOff;
-}
-
-size_t yajl_lex_current_char(yajl_lexer lexer)
-{
-    return lexer->charOff;
-}
-
-yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText,
-                       size_t jsonTextLen, size_t offset)
-{
-    const unsigned char * outBuf;
-    size_t outLen;
-    size_t bufLen = yajl_buf_len(lexer->buf);
-    size_t bufOff = lexer->bufOff;
-    unsigned int bufInUse = lexer->bufInUse;
-    yajl_tok tok;
-    
-    tok = yajl_lex_lex(lexer, jsonText, jsonTextLen, &offset,
-                       &outBuf, &outLen);
-
-    lexer->bufOff = bufOff;
-    lexer->bufInUse = bufInUse;
-    yajl_buf_truncate(lexer->buf, bufLen);
-    
-    return tok;
-}