edn_turbo 0.3.1 → 0.3.2

data/ext/edn_turbo/edn_parser.rl

@@ -23,9 +23,7 @@
         comment        = ';' cr_neg* counter;
         ignore         = ws | comment;
 
-        operators      = [/\.\*!_\?$%&<>\=+\-];
-        symbol_start   = alpha;
-        symbol_chars   = symbol_start | digit | [\#:_\-\.\'];
+        operators      = [/\.\*!_\?$%&<>\=+\-\'];
 
         begin_dispatch = '#';
         begin_keyword  = ':';
@@ -37,10 +35,7 @@
         string_delim   = '"';
         begin_number   = digit;
         begin_value    = alnum | [:\"\{\[\(\\\#^] | operators;
-        begin_symbol   = symbol_start;
-
-        symbol_name    = symbol_start (symbol_chars)*;
-        symbol         = (symbol_name ('/' symbol_name)?);
+        begin_symbol   = alpha;
 
         # int / decimal rules
         integer        = ('0' | [1-9] digit*);
@@ -68,19 +63,19 @@
 
     write data;
 
-    action parse_string {
+    action parse_val_string {
         // string types within double-quotes
         const char *np = parse_string(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_keyword {
+    action parse_val_keyword {
         // tokens with a leading ':'
         const char *np = parse_keyword(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_number {
+    action parse_val_number {
         // tokens w/ leading digits: non-negative integers & decimals.
         // try to parse a decimal first
         const char *np = parse_decimal(fpc, pe, v);
@@ -95,64 +90,64 @@
             fbreak;
         }
         else {
-            error(__FUNCTION__, *p);
+            error(__FUNCTION__, "number format error", *p);
             fexec pe;
         }
     }
 
-    action parse_operator {
+    action parse_val_operator {
         // stand-alone operators *, +, -, etc.
         const char *np = parse_operator(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_char {
+    action parse_val_char {
         // tokens w/ leading \ (escaped characters \newline, \c, etc.)
         const char *np = parse_esc_char(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_symbol {
+    action parse_val_symbol {
         // user identifiers and reserved keywords (true, false, nil)
         VALUE sym = Qnil;
         const char *np = parse_symbol(fpc, pe, sym);
-        if (np == NULL) { fhold; fbreak; } else {
+        if (np == NULL) { fexec pe; } else {
             // parse_symbol will make 'sym' a ruby string
             if      (std::strcmp(RSTRING_PTR(sym), "true") == 0)  { v = Qtrue; }
             else if (std::strcmp(RSTRING_PTR(sym), "false") == 0) { v = Qfalse; }
             else if (std::strcmp(RSTRING_PTR(sym), "nil") == 0)   { v = Qnil; }
             else {
-                v = Parser::make_edn_symbol(sym);
+                v = Parser::make_edn_type(EDNT_MAKE_SYMBOL_METHOD, sym);
             }
             fexec np;
         }
     }
 
-    action parse_vector {
+    action parse_val_vector {
         // [
         const char *np = parse_vector(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_list {
+    action parse_val_list {
         // (
         const char *np = parse_list(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_map {
+    action parse_val_map {
         // {
         const char *np = parse_map(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_meta {
+    action parse_val_meta {
         // ^
         const char *np = parse_meta(fpc, pe);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_dispatch {
+    action parse_val_dispatch {
         // handles tokens w/ leading # ("#_", "#{", and tagged elems)
         const char *np = parse_dispatch(fpc + 1, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
@@ -160,23 +155,24 @@
 
 
     main := (
-             string_delim >parse_string |
-             begin_keyword >parse_keyword |
-             begin_number >parse_number |
-             operators >parse_operator |
-             begin_char >parse_char |
-             begin_symbol >parse_symbol |
-             begin_vector >parse_vector |
-             begin_list >parse_list |
-             begin_map >parse_map |
-             begin_meta >parse_meta |
-             begin_dispatch >parse_dispatch
+             string_delim >parse_val_string |
+             begin_keyword >parse_val_keyword |
+             begin_number >parse_val_number |
+             operators >parse_val_operator |
+             begin_char >parse_val_char |
+             begin_symbol >parse_val_symbol |
+             begin_vector >parse_val_vector |
+             begin_list >parse_val_list |
+             begin_map >parse_val_map |
+             begin_meta >parse_val_meta |
+             begin_dispatch >parse_val_dispatch
             ) %*exit;
 }%%
 
 
 const char *edn::Parser::parse_value(const char *p, const char *pe, VALUE& v)
 {
+    //    std::cerr << __FUNCTION__ << "() p: \"" << p << "\"" << std::endl;
     int cs;
 
     %% write init;
@@ -186,7 +182,7 @@ const char *edn::Parser::parse_value(const char *p, const char *pe, VALUE& v)
         return p;
     }
     else if (cs == EDN_value_error) {
-        error(__FUNCTION__, *p);
+        error(__FUNCTION__, "token error", *p);
         return pe;
     }
     else if (cs == EDN_value_en_main) {} // silence ragel warning
@@ -207,7 +203,7 @@ const char *edn::Parser::parse_value(const char *p, const char *pe, VALUE& v)
 
     write data;
 
-    action parse_string {
+    action parse_chars {
         if (Parser::parse_byte_stream(p_save + 1, p, v, encode)) {
             fexec p + 1;
         } else {
@@ -220,11 +216,11 @@ const char *edn::Parser::parse_value(const char *p, const char *pe, VALUE& v)
     }
 
     main := string_delim (
-                          (^([\"\\] | 0..0x1f | 0xc2..0xf5) |
+                          (^([\"\\] | 0xc2..0xf5) |
                            ((0xc2..0xf5) |
                             '\\'[\"\\/bfnrt] |
                             '\\u'[0-9a-fA-F]{4}) $mark_for_encoding |
-                           '\\'^([\"\\/bfnrtu]|0..0x1f))* %parse_string
+                           '\\'^([\"\\/bfnrtu]))* %parse_chars
                           ) :>> string_delim @err(close_err) @exit;
 }%%
 
@@ -258,15 +254,16 @@ const char* edn::Parser::parse_string(const char *p, const char *pe, VALUE& v)
     machine EDN_keyword;
     include EDN_common;
 
-    keyword_chars = symbol_chars | operators;
-    keyword_start = symbol_start | [\#\./];
+    keyword_start = alpha | [\.\*!_\?$%&<>\=+\-\'\#];
+    keyword_chars = (keyword_start | digit | ':');
 
-    keyword_name    = keyword_start (keyword_chars)*;
+    keyword_name  = keyword_start keyword_chars*;
+    keyword       = keyword_name ('/' keyword_chars*)?;
 
     write data;
 
 
-    main := begin_keyword keyword_name (^keyword_chars? @exit);
+    main := begin_keyword keyword (^(keyword_chars | '/')? @exit);
 }%%
 
 
@@ -287,7 +284,7 @@ const char* edn::Parser::parse_keyword(const char *p, const char *pe, VALUE& v)
         return p;
     }
     else if (cs == EDN_keyword_error) {
-        error(__FUNCTION__, *p);
+        error(__FUNCTION__, "invalid keyword", *p);
         return pe;
     }
     else if (cs == EDN_keyword_en_main) {} // silence ragel warning
@@ -331,7 +328,7 @@ const char* edn::Parser::parse_decimal(const char *p, const char *pe, VALUE& v)
 
 
 // ============================================================
-// integer parsing machine
+// integer parsing machine - M suffix will return a BigNum
 //
 %%{
     machine EDN_integer;
@@ -376,17 +373,18 @@ const char* edn::Parser::parse_integer(const char *p, const char *pe, VALUE& v)
 
     write data;
 
-    action parse_symbol {
+    action parse_op_symbol {
         // parse a symbol including the leading operator (-, +, .)
         VALUE sym = Qnil;
         const char *np = parse_symbol(p_save, pe, sym);
-        if (np == NULL) { fhold; fbreak; } else {
-            v = Parser::make_edn_symbol(sym);
+        if (np == NULL) { fexec pe; } else {
+            if (sym != Qnil)
+                v = Parser::make_edn_type(EDNT_MAKE_SYMBOL_METHOD, sym);
             fexec np;
         }
     }
 
-    action parse_number {
+    action parse_op_number {
         // parse a number with the leading symbol - this is slightly
         // different than the one within EDN_value since it includes
         // the leading - or +
@@ -404,24 +402,27 @@ const char* edn::Parser::parse_integer(const char *p, const char *pe, VALUE& v)
             fbreak;
         }
         else {
-            error(__FUNCTION__, *p);
+            error(__FUNCTION__, "number format error", *p);
             fexec pe;
         }
     }
 
-    action parse_operator {
+    action parse_op {
         // stand-alone operators (-, +, /, ... etc)
         char op[2] = { *p_save, 0 };
         VALUE sym = rb_str_new2(op);
-        v = Parser::make_edn_symbol(sym);
+        v = Parser::make_edn_type(EDNT_MAKE_SYMBOL_METHOD, sym);
     }
 
+    valid_non_numeric_chars = alpha|operators|':'|'#';
+    valid_chars             = valid_non_numeric_chars | digit;
 
     main := (
-             ('-'|'+'|'.') alpha >parse_symbol |
-             ('-'|'+') begin_number >parse_number |
-             operators ignore* >parse_operator
-             ) ^(operators|alpha|digit)? @exit;
+             ('-'|'+') begin_number >parse_op_number |
+             (operators - [\-\+\.]) valid_chars >parse_op_symbol |
+             [\-\+\.] valid_non_numeric_chars valid_chars* >parse_op_symbol |
+             operators ignore* >parse_op
+             ) ^(valid_chars)? @exit;
 }%%
 
 
@@ -437,7 +438,7 @@ const char* edn::Parser::parse_operator(const char *p, const char *pe, VALUE& v)
         return p;
     }
     else if (cs == EDN_operator_error) {
-        error(__FUNCTION__, *p);
+        error(__FUNCTION__, "symbol syntax error", *p);
         return pe;
     }
     else if (cs == EDN_operator_en_main) {} // silence ragel warning
@@ -455,12 +456,13 @@ const char* edn::Parser::parse_operator(const char *p, const char *pe, VALUE& v)
 
     write data;
 
-    valid_chars = alpha;
+    valid_chars = extend;
 
 
-    main := (
-             begin_char valid_chars+ ignore*
-             ) (^(valid_chars | '\\')? @exit);
+    main := begin_char (
+                        'space' | 'newline' | 'tab' | 'return' | 'formfeed' | 'backspace' |
+                        valid_chars
+                        ) (ignore* | [\\\]\}\)])? @exit;
 }%%
 
 
@@ -480,7 +482,7 @@ const char* edn::Parser::parse_esc_char(const char *p, const char *pe, VALUE& v)
         return p;
     }
     else if (cs == EDN_escaped_char_error) {
-        error(__FUNCTION__, *p);
+        error(__FUNCTION__, "unexpected value", *p);
         return pe;
     }
     else if (cs == EDN_escaped_char_en_main) {} // silence ragel warning
@@ -502,10 +504,26 @@ const char* edn::Parser::parse_esc_char(const char *p, const char *pe, VALUE& v)
 
     write data;
 
+    symbol_ops_1 = [\.\-\+];
+    symbol_ops_2 = [\*!_\?$%&<>\=\'];
+    symbol_ops_3 = [:\#];
+
+    symbol_start = alpha | symbol_ops_1 | symbol_ops_2;
+
+    symbol_chars = symbol_start | digit | symbol_ops_3;
+
+    symbol_name  = (
+                    (alpha symbol_chars*) |
+                    (symbol_ops_1 (symbol_start | symbol_ops_3) symbol_chars*) |
+                    (symbol_start symbol_chars+) |
+                    operators{1}
+                    );
+    symbol       = '/' | (symbol_name ('/' symbol_name)?);
+
 
     main := (
-             operators? symbol
-             ) ignore* (^(symbol_chars | operators)? @exit);
+             symbol
+             ) ignore* (^(symbol_chars | '/')? @exit);
 }%%
 
 
@@ -525,8 +543,7 @@ const char* edn::Parser::parse_symbol(const char *p, const char *pe, VALUE& s)
         return p;
     }
     else if (cs == EDN_symbol_error) {
-        error(__FUNCTION__, *p);
-        return pe;
+        error(__FUNCTION__, "invalid symbol sequence", *p);
     }
     else if (cs == EDN_symbol_en_main) {} // silence ragel warning
     return NULL;
@@ -544,25 +561,48 @@ const char* edn::Parser::parse_symbol(const char *p, const char *pe, VALUE& s)
     machine EDN_sequence_common;
     include EDN_common;
 
+    action open_seq {
+        // sequences store elements in an array, then process it to
+        // convert it to a list, set, or map as needed once the
+        // sequence end is reached
+        elems = rb_ary_new();
+        // additionally, metadata for elements in the sequence may be
+        // carried so we must push a new level in the metadata stack
+        new_meta_list();
+    }
+
+    action close_seq {
+        // remove the current metadata level
+        del_top_meta_list();
+    }
+
     action parse_item {
         // reads an item within a sequence (vector, list, map, or
         // set). Regardless of the sequence type, an array of the
         // items is built. Once done, the sequence parser will convert
         // if needed
         VALUE e;
+        std::size_t meta_sz = meta_size();
         const char *np = parse_value(fpc, pe, e);
-        if (np == NULL) {
-            fhold; fbreak;
-        } else {
+        if (np == NULL) { fhold; fbreak; } else {
             // if there's an entry in the discard list, the current
             // object is not meant to be kept due to a #_ so don't
             // push it into the list of elements
             if (!discard.empty()) {
                 discard.pop_back();
             }
-            else {
-                // otherwise we add it to the list of elements for the
-                // corresponding container
+            else if (!meta_empty()) {
+                // check if parse_value added metadata
+                if (meta_size() == meta_sz) {
+                    // there's metadata and it didn't increase so
+                    // parse_value() read an element we care
+                    // about. Bind the metadata to it and add it to
+                    // the sequence
+                    e = Parser::make_edn_type(EDNT_EXTENDED_VALUE_METHOD, e, ruby_meta());
+                    rb_ary_push(elems, e);
+                }
+            } else {
+                // no metadata.. just push it
                 rb_ary_push(elems, e);
             }
             fexec np;
@@ -584,10 +624,9 @@ const char* edn::Parser::parse_symbol(const char *p, const char *pe, VALUE& s)
 
     write data;
 
-    main := begin_vector (
-                          ignore* sequence? :>> end_vector
-                          )
-                          @err(close_err) @exit;
+    main := begin_vector @open_seq (
+                                    ignore* sequence? :>> end_vector @close_seq
+                                    ) @err(close_err) @exit;
 }%%
 
 
@@ -599,7 +638,7 @@ const char* edn::Parser::parse_vector(const char *p, const char *pe, VALUE& v)
     static const char* EDN_TYPE = "vector";
 
     int cs;
-    VALUE elems = rb_ary_new(); // will store the vector's elements
+    VALUE elems; // will store the vector's elements - allocated in @open_seq
 
     %% write init;
     %% write exec;
@@ -609,7 +648,7 @@ const char* edn::Parser::parse_vector(const char *p, const char *pe, VALUE& v)
         return p + 1;
     }
     else if (cs == EDN_vector_error) {
-        error(__FUNCTION__, *p);
+        error(__FUNCTION__, "vector format error", *p);
         return pe;
     }
     else if (cs == EDN_vector_en_main) {} // silence ragel warning
@@ -629,9 +668,9 @@ const char* edn::Parser::parse_vector(const char *p, const char *pe, VALUE& v)
 
     write data;
 
-    main := begin_list (
-                        ignore* sequence? :>> end_list
-                        ) @err(close_err) @exit;
+    main := begin_list @open_seq (
+                                  ignore* sequence? :>> end_list @close_seq
+                                  ) @err(close_err) @exit;
 }%%
 
 //
@@ -642,13 +681,15 @@ const char* edn::Parser::parse_list(const char *p, const char *pe, VALUE& v)
     static const char* EDN_TYPE = "list";
 
     int cs;
-    VALUE elems = rb_ary_new();
+    VALUE elems; // stores the list's elements - allocated in @open_seq
 
     %% write init;
     %% write exec;
 
     if (cs >= EDN_list_first_final) {
         v = elems;
+        // TODO: replace with this but first figure out why array is not unrolled by EDN::list()
+        //        v = Parser::make_edn_type(EDNT_MAKE_LIST_METHOD, elems);
         return p + 1;
     }
     else if (cs == EDN_list_error) {
@@ -673,9 +714,9 @@ const char* edn::Parser::parse_list(const char *p, const char *pe, VALUE& v)
     write data;
 
 
-    main := begin_map (
-                       ignore* (sequence)? :>> end_map
-                       ) @err(close_err) @exit;
+    main := begin_map @open_seq (
+                                 ignore* (sequence)? :>> end_map @close_seq
+                                 ) @err(close_err) @exit;
 }%%
 
 
@@ -685,8 +726,8 @@ const char* edn::Parser::parse_map(const char *p, const char *pe, VALUE& v)
 
     int cs;
     // since we don't know whether we're looking at a key or value,
-    // initially store all elements in a list
-    VALUE elems = rb_ary_new();
+    // initially store all elements in an array (allocated in @open_seq)
+    VALUE elems;
 
     %% write init;
     %% write exec;
@@ -730,19 +771,19 @@ const char* edn::Parser::parse_map(const char *p, const char *pe, VALUE& v)
 
     write data;
 
-    action parse_set {
+    action parse_disp_set {
         // #{ }
         const char *np = parse_set(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_discard {
+    action parse_disp_discard {
         // discard token #_
         const char *np = parse_discard(fpc, pe);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
-    action parse_tagged {
+    action parse_disp_tagged {
         // #inst, #uuid, or #user/tag
         const char *np = parse_tagged(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
@@ -750,9 +791,9 @@ const char* edn::Parser::parse_map(const char *p, const char *pe, VALUE& v)
 
 
     main := (
-             ('{' >parse_set |
-              '_' >parse_discard |
-              alpha >parse_tagged)
+             ('{' >parse_disp_set |
+              '_' >parse_disp_discard |
+              alpha >parse_disp_tagged)
              ) @exit;
 }%%
 
@@ -768,7 +809,7 @@ const char* edn::Parser::parse_dispatch(const char *p, const char *pe, VALUE& v)
         return p + 1;
     }
     else if (cs == EDN_dispatch_error) {
-        error(__FUNCTION__, *p);
+        error(__FUNCTION__, "dispatch extend error", *p);
         return pe;
     }
     else if (cs == EDN_dispatch_en_main) {} // silence ragel warning
@@ -789,9 +830,9 @@ const char* edn::Parser::parse_dispatch(const char *p, const char *pe, VALUE& v)
     begin_set    = '{';
     end_set      = '}';
 
-    main := begin_set (
-                       ignore* sequence? :>> end_set
-                       ) @err(close_err) @exit;
+    main := begin_set @open_seq (
+                                 ignore* sequence? :>> end_set @close_seq
+                                 ) @err(close_err) @exit;
 }%%
 
 //
@@ -802,14 +843,14 @@ const char* edn::Parser::parse_set(const char *p, const char *pe, VALUE& v)
     static const char* EDN_TYPE = "set";
 
     int cs;
-    VALUE elems = rb_ary_new(); // stored as an array
+    VALUE elems; // holds the set's elements as an array allocated in @open_seq
 
     %% write init;
     %% write exec;
 
     if (cs >= EDN_set_first_final) {
         // all elements collected; now convert to a set
-        v = Parser::make_ruby_set(elems);
+        v = Parser::make_edn_type(EDNT_MAKE_SET_METHOD, elems);
         return p + 1;
     }
     else if (cs == EDN_set_error) {
@@ -837,14 +878,12 @@ const char* edn::Parser::parse_set(const char *p, const char *pe, VALUE& v)
 
     action discard_value {
         const char *np = parse_value(fpc, pe, v);
-        if (np) {
-            // this token is to be discard it so store it in the
+        if (np == NULL) { fhold; fbreak; } else {
+            // this token is to be discarded so store it in the
             // discard stack - we really don't need to save it so this
             // could be simplified
             discard.push_back(v);
             fexec np;
-        } else {
-            fhold; fbreak;
         }
     }
 
@@ -901,24 +940,41 @@ const char* edn::Parser::parse_discard(const char *p, const char *pe)
     machine EDN_tagged;
     include EDN_common;
 
+    write data;
+
+    tag_symbol_chars_start       = alpha;
+    tag_symbol_chars_non_numeric = tag_symbol_chars_start | [\.\*!_\?$%&<>\=+\-\'\:\#];
+    tag_symbol_chars             = tag_symbol_chars_non_numeric | digit;
+
+    tag_symbol_namespace         = tag_symbol_chars_start (tag_symbol_chars)*;
+    tag_symbol_name              = tag_symbol_chars_non_numeric (tag_symbol_chars)*;
+
+    tag_symbol                   = (tag_symbol_namespace ('/' tag_symbol_name)?);
+
 #    inst = (string_delim [0-9+\-:\.TZ]* string_delim);
 #    uuid = (string_delim [a-f0-9\-]* string_delim);
 
-    write data;
-
-    action parse_symbol {
+    action parse_tag {
         // parses the symbol portion of the pair
         const char *np = parse_symbol(fpc, pe, sym_name);
-        if (np == NULL) { fhold; fbreak; } else { fexec np; }
+        if (np == NULL) { fhold; fbreak; } else {
+            sym_ok = true;
+            fexec np;
+        }
     }
-    action parse_value {
+    action parse_data {
         // parses the value portion
         const char *np = parse_value(fpc, pe, data);
-        if (np == NULL) { fhold; fbreak; } else { fexec np; }
+        if (np == NULL) { fhold; fbreak; } else {
+            data_ok = true;
+            fexec np;
+        }
     }
 
-
-    main := (symbol >parse_symbol ignore* begin_value >parse_value) @exit;
+    main := (
+             tag_symbol >parse_tag ignore+
+             begin_value >parse_data
+             ) @exit;
 }%%
 
 
@@ -926,6 +982,8 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, VALUE& v)
 {
     VALUE sym_name = Qnil;
     VALUE data = Qnil;
+    bool sym_ok = false;
+    bool data_ok = false;
 
     int cs;
 
@@ -935,20 +993,27 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, VALUE& v)
     if (cs >= EDN_tagged_first_final) {
         //std::cerr << __FUNCTION__ << " parse symbol name as '" << sym_name << "', value is: " << data << std::endl;
 
+        if (!sym_ok || !data_ok) {
+            error(__FUNCTION__, "tagged element symbol error", *p);
+            v =  EDNT_EOF_CONST;
+            return NULL;
+        }
+
         try {
             // tagged_element makes a call to ruby which may throw an
             // exception when parsing the data
-            v = Parser::tagged_element(sym_name, data);
+            v = Parser::make_edn_type(EDNT_TAGGED_ELEM_METHOD, sym_name, data);
+            return p + 1;
         } catch (std::exception& e) {
             error(__FUNCTION__, e.what());
             return pe;
         }
-        return p + 1;
     }
     else if (cs == EDN_tagged_error) {
-        return pe;
+        error(__FUNCTION__, "tagged element symbol error", *p);
     }
     else if (cs == EDN_tagged_en_main) {} // silence ragel warning
+    v =  EDNT_EOF_CONST;
     return NULL;
 }
 
@@ -966,13 +1031,13 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, VALUE& v)
 
     write data;
 
-    action parse_meta {
+    action parse_data {
         const char *np = parse_value(fpc, pe, v);
-        if (np) { fexec np; } else { fhold; fbreak; }
+        if (np == NULL) { fhold; fbreak; } else { fexec np; }
     }
 
     main := begin_meta (
-                        begin_value >parse_meta
+                        begin_value >parse_data
                         ) @exit;
 }%%
 
@@ -986,7 +1051,7 @@ const char* edn::Parser::parse_meta(const char *p, const char *pe)
     %% write exec;
 
     if (cs >= EDN_meta_first_final) {
-        metadata.push_back(v);
+        append_to_meta(v);
         return p + 1;
     }
     else if (cs == EDN_meta_error) {
@@ -1010,25 +1075,25 @@ const char* edn::Parser::parse_meta(const char *p, const char *pe)
 
     write data;
 
-    action parse_value {
+    action parse_elem {
         // save the count of metadata items before we parse this value
         // so we can determine if we've read another metadata value or
         // an actual data item
-        std::size_t meta_size = metadata.size();
+        std::size_t meta_sz = meta_size();
         const char* np = parse_value(fpc, pe, result);
         if (np == NULL) { fexec pe; fbreak; } else {
             // if we have metadata saved and it matches the count we
             // saved before we parsed a value, then we must bind the
             // metadata sequence to it
-            if (!metadata.empty() && metadata.size() == meta_size) {
+            if (!meta_empty() && meta_size() == meta_sz) {
                 // this will empty the metadata sequence too
-                result = bind_meta_to_value(result);
+                result = Parser::make_edn_type(EDNT_EXTENDED_VALUE_METHOD, result, ruby_meta());
             }
             fexec np;
         }
     }
 
-    element       = begin_value >parse_value;
+    element       = begin_value >parse_elem;
     next_element  = ignore* element;
     sequence      = ((element ignore*) (next_element ignore*)*);
 
@@ -1039,16 +1104,15 @@ const char* edn::Parser::parse_meta(const char *p, const char *pe)
 VALUE edn::Parser::parse(const char* src, std::size_t len)
 {
     int cs;
-    VALUE result = Qnil;
+    VALUE result = EDNT_EOF_CONST;
 
     %% write init;
     set_source(src, len);
     %% write exec;
 
     if (cs == EDN_parser_error) {
-        if (p)
-            error(__FUNCTION__, *p);
-        return EDNT_EOF;
+        error(__FUNCTION__, *p);
+        return EDNT_EOF_CONST;
     }
     else if (cs == EDN_parser_first_final) {
         p = pe = eof = NULL;
@@ -1065,50 +1129,52 @@ VALUE edn::Parser::parse(const char* src, std::size_t len)
     machine EDN_tokens;
     include EDN_common;
 
-    write data nofinal;
+    write data nofinal noerror;
 
-    action parse_value {
+    action parse_token {
         // we won't know if we've parsed a discard or a metadata until
         // after parse_value() is done. Save the current number of
         // elements in the metadata sequence; then we can check if it
         // grew or if the discard sequence grew
-        meta_size = metadata.size();
+        meta_sz = meta_size();
 
         const char* np = parse_value(fpc, pe, value);
-
         if (np == NULL) { fhold; fbreak; } else {
-            if (metadata.size() > 0) {
-                // was anotheran additional metadata entry read? if
-                // so, don't return a value
-                if (metadata.size() > meta_size) {
-                    is_value = false;
+            if (!meta_empty()) {
+                // was an additional metadata entry read? if so, don't
+                // return a value
+                if (meta_size() > meta_sz) {
+                    state = TOKEN_IS_META;
                 }
                 else {
                     // a value was read and there's a pending metadata
                     // sequence. Bind them.
-                    value = bind_meta_to_value(value);
+                    value = Parser::make_edn_type(EDNT_EXTENDED_VALUE_METHOD, value, ruby_meta());
+                    state = TOKEN_OK;
                 }
             } else if (!discard.empty()) {
                 // a discard read. Don't return a value
-                is_value = false;
+                state = TOKEN_IS_DISCARD;
+            } else {
+                state = TOKEN_OK;
             }
             fexec np;
         }
     }
 
-    main := ignore* begin_value >parse_value ignore*;
+    main := ignore* begin_value >parse_token ignore*;
 }%%
 
 
 //
 //
-bool edn::Parser::parse_next(VALUE& value)
+edn::Parser::eTokenState edn::Parser::parse_next(VALUE& value)
 {
     int cs;
-    bool is_value = true;
+    eTokenState state = TOKEN_ERROR;
     // need to track metadada read and bind it to the next value read
     // - but must account for sequences of metadata values
-    std::size_t meta_size;
+    std::size_t meta_sz;
 
     // clear any previously saved discards; only track if read during
     // this op
@@ -1117,12 +1183,8 @@ bool edn::Parser::parse_next(VALUE& value)
     %% write init;
     %% write exec;
 
-    if (cs == EDN_parser_error) {
-        value = EDNT_EOF;
-    }
-    else if (cs == EDN_tokens_en_main) {} // silence ragel warning
-
-    return is_value;
+    if (cs == EDN_tokens_en_main) {} // silence ragel warning
+    return state;
 }