edn_turbo 0.2.2 → 0.3.0

data/ext/edn_turbo/edn_parser.h

@@ -5,18 +5,12 @@
 #include <sstream>
 #include <stack>
 
-#include <rice/Module.hpp>
-#include <rice/Object.hpp>
-#include <rice/Array.hpp>
-#include <rice/String.hpp>
-#include <rice/to_from_ruby.hpp>
-
-typedef unsigned char ui8;
+#include <ruby/ruby.h>
 
 
 namespace edn
 {
-    extern Rice::Module rb_mEDNT;
+    extern VALUE rb_mEDNT;
     extern VALUE EDNT_MAKE_EDN_SYMBOL;
     extern VALUE EDNT_MAKE_SET_METHOD;
     extern VALUE EDNT_TAGGED_ELEM;
@@ -27,65 +21,72 @@ namespace edn
     // C-extension EDN Parser class representation
     class Parser
     {
-    private:
+    public:
+        Parser() : p(NULL), pe(NULL), eof(NULL), line_number(1) { }
 
-        std::size_t line_number;
-        const char* p_save;
+        void set_source(const char* src, std::size_t len);
+
+        bool is_eof() const { return (p != NULL && p == pe); }
+        VALUE parse(const char* s, std::size_t len);
+        //        VALUE read(const std::string& data) { return parse(data.c_str(), data.length()); }
+        VALUE next();
+
+        static void throw_error(int error);
+
+    private:
+        // ragel needs these
+        const char* p;
+        const char* pe;
         const char* eof;
-        std::stack<Rice::Object> discard;
-
-        Rice::Object parse(const char* s, std::size_t len);
-
-        const char* parse_value   (const char *p, const char *pe, Rice::Object& o);
-        const char* parse_string  (const char *p, const char *pe, Rice::Object& o);
-        const char* parse_keyword (const char *p, const char *pe, Rice::Object& o);
-        const char* parse_decimal (const char *p, const char *pe, Rice::Object& o);
-        const char* parse_integer (const char *p, const char *pe, Rice::Object& o);
-        const char* parse_operator(const char *p, const char *pe, Rice::Object& o);
-        const char* parse_esc_char(const char *p, const char *pe, Rice::Object& o);
-        const char* parse_symbol  (const char *p, const char *pe, std::string& s);
-        const char* parse_vector  (const char *p, const char *pe, Rice::Object& o);
-        const char* parse_list    (const char *p, const char *pe, Rice::Object& o);
-        const char* parse_map     (const char *p, const char *pe, Rice::Object& o);
-        const char* parse_dispatch(const char *p, const char *pe, Rice::Object& o);
-        const char* parse_set     (const char *p, const char *pe, Rice::Object& o);
+        std::size_t line_number;
+        std::stack<VALUE> discard;
+
+        void reset();
+
+        const char* parse_value   (const char *p, const char *pe, VALUE& v);
+        const char* parse_string  (const char *p, const char *pe, VALUE& v);
+        const char* parse_keyword (const char *p, const char *pe, VALUE& v);
+        const char* parse_decimal (const char *p, const char *pe, VALUE& v);
+        const char* parse_integer (const char *p, const char *pe, VALUE& v);
+        const char* parse_operator(const char *p, const char *pe, VALUE& v);
+        const char* parse_esc_char(const char *p, const char *pe, VALUE& v);
+        const char* parse_symbol  (const char *p, const char *pe, VALUE& v);
+        const char* parse_vector  (const char *p, const char *pe, VALUE& v);
+        const char* parse_list    (const char *p, const char *pe, VALUE& v);
+        const char* parse_map     (const char *p, const char *pe, VALUE& v);
+        const char* parse_dispatch(const char *p, const char *pe, VALUE& v);
+        const char* parse_set     (const char *p, const char *pe, VALUE& v);
         const char* parse_discard (const char *p, const char *pe);
-        const char* parse_tagged  (const char *p, const char *pe, Rice::Object& o);
+        const char* parse_tagged  (const char *p, const char *pe, VALUE& v);
 
         // defined in edn_parser_unicode.cc
         static bool to_utf8(const char *s, std::size_t len, std::string& rslt);
 
         // defined in edn_parser_util.cc
-        static Rice::Object integer_to_ruby(const char* str, std::size_t len);
-        static Rice::Object float_to_ruby  (const char* str, std::size_t len);
+        static VALUE integer_to_ruby(const char* str, std::size_t len);
+        static VALUE float_to_ruby  (const char* str, std::size_t len);
 
-        static bool parse_byte_stream (const char *p, const char *pe, Rice::String& s, bool encode);
-        static bool parse_escaped_char(const char *p, const char *pe, Rice::Object& s);
+        static bool parse_byte_stream (const char *p, const char *pe, VALUE& rslt, bool encode);
+        static bool parse_escaped_char(const char *p, const char *pe, VALUE& rslt);
 
-        static Rice::Object make_edn_symbol(const std::string& name);
-        static Rice::Object make_ruby_set(const Rice::Array& elems);
-        static Rice::Object tagged_element(const std::string& name, const Rice::Object& data);
-
-        void error(const std::string& f, const std::string& err, char c) const;
-        void error(const std::string& f, char err_c) const { error(f, "", err_c); }
-        void error(const std::string& f, const std::string& err_msg) const { error(f, err_msg, '\0'); }
+        static VALUE make_edn_symbol(VALUE sym);
+        static VALUE make_ruby_set(VALUE elems);
+        static VALUE tagged_element(VALUE name, VALUE data);
 
         // utility method to convert a primitive in string form to a
         // ruby type
         template <class T>
-        static Rice::Object buftotype(const char* p, std::size_t len) {
+        static inline T buftotype(const char* p, std::size_t len) {
             T val;
             std::string buf;
             buf.append(p, len);
             std::istringstream(buf) >> val;
-            return to_ruby<T>(val);
+            return val;
         }
 
-    public:
-        Parser() : line_number(1), p_save(NULL), eof(NULL) { }
-
-        Rice::Object process(const std::string& data) { return parse(data.c_str(), data.length()); }
-
+        void error(const std::string& f, const std::string& err, char c) const;
+        void error(const std::string& f, char err_c) const { error(f, "", err_c); }
+        void error(const std::string& f, const std::string& err_msg) const { error(f, err_msg, '\0'); }
     }; // Engine
 
 } // namespace

data/ext/edn_turbo/edn_parser.rl

@@ -1,11 +1,7 @@
 #include <iostream>
 #include <string>
 #include <stack>
-
-#include <rice/Hash.hpp>
-#include <rice/Array.hpp>
-#include <rice/to_from_ruby.hpp>
-#include <rice/Exception.hpp>
+#include <exception>
 
 #include "edn_parser.h"
 
@@ -73,23 +69,23 @@
 
     action parse_string {
         // string types within double-quotes
-        const char *np = parse_string(fpc, pe, o);
+        const char *np = parse_string(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
     action parse_keyword {
         // tokens with a leading ':'
-        const char *np = parse_keyword(fpc, pe, o);
+        const char *np = parse_keyword(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
     action parse_number {
         // tokens w/ leading digits: non-negative integers & decimals.
         // try to parse a decimal first
-        const char *np = parse_decimal(fpc, pe, o);
+        const char *np = parse_decimal(fpc, pe, v);
         if (np == NULL) {
             // if we can't, try to parse it as an int
-            np = parse_integer(fpc, pe, o);
+            np = parse_integer(fpc, pe, v);
         }
 
         if (np) {
@@ -105,26 +101,27 @@
 
     action parse_operator {
         // stand-alone operators *, +, -, etc.
-        const char *np = parse_operator(fpc, pe, o);
+        const char *np = parse_operator(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
     action parse_char {
         // tokens w/ leading \ (escaped characters \newline, \c, etc.)
-        const char *np = parse_esc_char(fpc, pe, o);
+        const char *np = parse_esc_char(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
     action parse_symbol {
         // user identifiers and reserved keywords (true, false, nil)
-        std::string sym;
+        VALUE sym = Qnil;
         const char *np = parse_symbol(fpc, pe, sym);
         if (np == NULL) { fhold; fbreak; } else {
-            if      (sym == "true")  { o = Qtrue; }
-            else if (sym == "false") { o = Qfalse; }
-            else if (sym == "nil")   { o = Qnil; }
+            // parse_symbol will make 'sym' a ruby string
+            if      (std::strcmp(RSTRING_PTR(sym), "true") == 0)  { v = Qtrue; }
+            else if (std::strcmp(RSTRING_PTR(sym), "false") == 0) { v = Qfalse; }
+            else if (std::strcmp(RSTRING_PTR(sym), "nil") == 0)   { v = Qnil; }
             else {
-                o = Parser::make_edn_symbol(sym);
+                v = Parser::make_edn_symbol(sym);
             }
             fexec np;
         }
@@ -132,25 +129,25 @@
 
     action parse_vector {
         // [
-        const char *np = parse_vector(fpc, pe, o);
+        const char *np = parse_vector(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
     action parse_list {
         // (
-        const char *np = parse_list(fpc, pe, o);
+        const char *np = parse_list(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
     action parse_map {
         // {
-        const char *np = parse_map(fpc, pe, o);
+        const char *np = parse_map(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
     action parse_dispatch {
         // handles tokens w/ leading # ("#_", "#{", and tagged elems)
-        const char *np = parse_dispatch(fpc + 1, pe, o);
+        const char *np = parse_dispatch(fpc + 1, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
@@ -170,7 +167,7 @@
 }%%
 
 
-const char *edn::Parser::parse_value(const char *p, const char *pe, Rice::Object& o)
+const char *edn::Parser::parse_value(const char *p, const char *pe, VALUE& v)
 {
     int cs;
 
@@ -203,7 +200,7 @@ const char *edn::Parser::parse_value(const char *p, const char *pe, Rice::Object
     write data;
 
     action parse_string {
-        if (Parser::parse_byte_stream(p_save + 1, p, s, encode)) {
+        if (Parser::parse_byte_stream(p_save + 1, p, v, encode)) {
             fexec p + 1;
         } else {
             fhold; fbreak;
@@ -224,21 +221,18 @@ const char *edn::Parser::parse_value(const char *p, const char *pe, Rice::Object
 }%%
 
 
-const char* edn::Parser::parse_string(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_string(const char *p, const char *pe, VALUE& v)
 {
     //    std::cerr << __FUNCTION__ << "   -  p: '" << p << "'" << std::endl;
     static const char* EDN_TYPE = "string";
     int cs;
     bool encode = false;
-    const char *eof = pe;
-    Rice::String s;
 
     %% write init;
-    p_save = p;
+    const char* p_save = p;
     %% write exec;
 
     if (cs >= EDN_string_first_final) {
-        o = s;
         return p + 1;
     }
     else if (cs == EDN_string_error) {
@@ -269,20 +263,20 @@ const char* edn::Parser::parse_string(const char *p, const char *pe, Rice::Objec
 }%%
 
 
-const char* edn::Parser::parse_keyword(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_keyword(const char *p, const char *pe, VALUE& v)
 {
     int cs;
 
     %% write init;
-    p_save = p;
+    const char* p_save = p;
     %% write exec;
 
     if (cs >= EDN_keyword_first_final) {
         std::string buf;
         uint32_t len = p - p_save;
-        // don't include leading ':' because Rice::Symbol will handle it
+        // don't include leading ':' because the ruby symbol will handle it
         buf.append(p_save + 1, len - 1);
-        o = Rice::Symbol(buf);
+        v = ID2SYM(rb_intern(buf.c_str()));
         return p;
     }
     else if (cs == EDN_keyword_error) {
@@ -312,16 +306,16 @@ const char* edn::Parser::parse_keyword(const char *p, const char *pe, Rice::Obje
 }%%
 
 
-const char* edn::Parser::parse_decimal(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_decimal(const char *p, const char *pe, VALUE& v)
 {
     int cs;
 
     %% write init;
-    p_save = p;
+    const char* p_save = p;
     %% write exec;
 
     if (cs >= EDN_decimal_first_final) {
-        o = Parser::float_to_ruby(p_save, p - p_save);
+        v = Parser::float_to_ruby(p_save, p - p_save);
         return p + 1;
     }
     else if (cs == EDN_decimal_en_main) {} // silence ragel warning
@@ -344,16 +338,16 @@ const char* edn::Parser::parse_decimal(const char *p, const char *pe, Rice::Obje
              ) (^[0-9MN+\-]? @exit);
 }%%
 
-const char* edn::Parser::parse_integer(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_integer(const char *p, const char *pe, VALUE& v)
 {
     int cs;
 
     %% write init;
-    p_save = p;
+    const char* p_save = p;
     %% write exec;
 
     if (cs >= EDN_integer_first_final) {
-        o = Parser::integer_to_ruby(p_save, p - p_save);
+        v = Parser::integer_to_ruby(p_save, p - p_save);
         return p + 1;
     }
     else if (cs == EDN_integer_en_main) {} // silence ragel warning
@@ -377,10 +371,10 @@ const char* edn::Parser::parse_integer(const char *p, const char *pe, Rice::Obje
 
     action parse_symbol {
         // parse a symbol including the leading operator (-, +, .)
-        std::string sym;
+        VALUE sym = Qnil;
         const char *np = parse_symbol(p_save, pe, sym);
         if (np == NULL) { fhold; fbreak; } else {
-            o = Parser::make_edn_symbol(sym);
+            v = Parser::make_edn_symbol(sym);
             fexec np;
         }
     }
@@ -391,10 +385,10 @@ const char* edn::Parser::parse_integer(const char *p, const char *pe, Rice::Obje
         // the leading - or +
         //
         // try to parse a decimal first
-        const char *np = parse_decimal(p_save, pe, o);
+        const char *np = parse_decimal(p_save, pe, v);
         if (np == NULL) {
             // if we can't, try to parse it as an int
-            np = parse_integer(p_save, pe, o);
+            np = parse_integer(p_save, pe, v);
         }
 
         if (np) {
@@ -410,9 +404,9 @@ const char* edn::Parser::parse_integer(const char *p, const char *pe, Rice::Obje
 
     action parse_operator {
         // stand-alone operators (-, +, /, ... etc)
-        std::string sym;
-        sym += *(p_save);
-        o = Parser::make_edn_symbol(sym);
+        char op[2] = { *p_save, 0 };
+        VALUE sym = rb_str_new2(op);
+        v = Parser::make_edn_symbol(sym);
     }
 
 
@@ -424,12 +418,12 @@ const char* edn::Parser::parse_integer(const char *p, const char *pe, Rice::Obje
 }%%
 
 
-const char* edn::Parser::parse_operator(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_operator(const char *p, const char *pe, VALUE& v)
 {
     int cs;
 
     %% write init;
-    p_save = p;
+    const char* p_save = p;
     %% write exec;
 
     if (cs >= EDN_operator_first_final) {
@@ -463,17 +457,17 @@ const char* edn::Parser::parse_operator(const char *p, const char *pe, Rice::Obj
 }%%
 
 
-const char* edn::Parser::parse_esc_char(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_esc_char(const char *p, const char *pe, VALUE& v)
 {
     int cs;
 
     %% write init;
-    p_save = p;
+    const char* p_save = p;
     %% write exec;
 
     if (cs >= EDN_escaped_char_first_final) {
         // convert the escaped value to a character
-        if (!Parser::parse_escaped_char(p_save + 1, p, o)) {
+        if (!Parser::parse_escaped_char(p_save + 1, p, v)) {
             return pe;
         }
         return p;
@@ -494,6 +488,7 @@ const char* edn::Parser::parse_esc_char(const char *p, const char *pe, Rice::Obj
 // character and an optional leading operator (name, -today,
 // .yesterday)
 //
+//
 %%{
     machine EDN_symbol;
     include EDN_common;
@@ -507,18 +502,19 @@ const char* edn::Parser::parse_esc_char(const char *p, const char *pe, Rice::Obj
 }%%
 
 
-const char* edn::Parser::parse_symbol(const char *p, const char *pe, std::string& sym)
+const char* edn::Parser::parse_symbol(const char *p, const char *pe, VALUE& s)
 {
     int cs;
 
     %% write init;
-    p_save = p;
+    const char* p_save = p;
     %% write exec;
 
     if (cs >= EDN_symbol_first_final) {
         // copy the symbol text
-        sym.clear();
-        sym.append(p_save, p - p_save);
+        if (s == Qnil)
+            s = rb_str_new2("");
+        rb_str_cat(s, p_save, p - p_save);
         return p;
     }
     else if (cs == EDN_symbol_error) {
@@ -533,7 +529,7 @@ const char* edn::Parser::parse_symbol(const char *p, const char *pe, std::string
 
 // ============================================================
 // EDN_sequence_common is used to parse EDN containers - elements are
-// initially stored in a rice array and then the final corresponding
+// initially stored in an array and then the final corresponding
 // container is built from the list (although, for vectors, lists, and
 // sets the same array is used)
 //
@@ -546,7 +542,7 @@ const char* edn::Parser::parse_symbol(const char *p, const char *pe, std::string
         // set). Regardless of the sequence type, an array of the
         // items is built. Once done, the sequence parser will convert
         // if needed
-        Rice::Object e;
+        VALUE e;
         const char *np = parse_value(fpc, pe, e);
         if (np == NULL) {
             fhold; fbreak;
@@ -560,7 +556,7 @@ const char* edn::Parser::parse_symbol(const char *p, const char *pe, std::string
             else {
                 // otherwise we add it to the list of elements for the
                 // corresponding container
-                elems.push(e);
+                rb_ary_push(elems, e);
             }
             fexec np;
         }
@@ -591,18 +587,18 @@ const char* edn::Parser::parse_symbol(const char *p, const char *pe, std::string
 //
 // vector parsing
 //
-const char* edn::Parser::parse_vector(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_vector(const char *p, const char *pe, VALUE& v)
 {
     static const char* EDN_TYPE = "vector";
 
     int cs;
-    Rice::Array elems; // will store the vector's elements
+    VALUE elems = rb_ary_new(); // will store the vector's elements
 
     %% write init;
     %% write exec;
 
     if (cs >= EDN_vector_first_final) {
-        o = elems;
+        v = elems;
         return p + 1;
     }
     else if (cs == EDN_vector_error) {
@@ -634,18 +630,18 @@ const char* edn::Parser::parse_vector(const char *p, const char *pe, Rice::Objec
 //
 // list parsing
 //
-const char* edn::Parser::parse_list(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_list(const char *p, const char *pe, VALUE& v)
 {
     static const char* EDN_TYPE = "list";
 
     int cs;
-    Rice::Array elems;
+    VALUE elems = rb_ary_new();
 
     %% write init;
     %% write exec;
 
     if (cs >= EDN_list_first_final) {
-        o = elems;
+        v = elems;
         return p + 1;
     }
     else if (cs == EDN_list_error) {
@@ -676,14 +672,14 @@ const char* edn::Parser::parse_list(const char *p, const char *pe, Rice::Object&
 }%%
 
 
-const char* edn::Parser::parse_map(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_map(const char *p, const char *pe, VALUE& v)
 {
     static const char* EDN_TYPE = "map";
 
     int cs;
     // since we don't know whether we're looking at a key or value,
     // initially store all elements in a list
-    Rice::Array elems;
+    VALUE elems = rb_ary_new();
 
     %% write init;
     %% write exec;
@@ -691,20 +687,20 @@ const char* edn::Parser::parse_map(const char *p, const char *pe, Rice::Object&
     if (cs >= EDN_map_first_final) {
 
         // hash parsing is done. Make sure we have an even count
-        if ((elems.size() % 2) != 0) {
+        if ((RARRAY_LEN(elems) % 2) != 0) {
             error(__FUNCTION__, "odd number of elements in map");
             return pe;
         }
 
         // now convert the sequence to a hash
-        Rice::Hash rslt;
-        while (elems.size())
+        VALUE rslt = rb_hash_new();
+        while (RARRAY_LEN(elems) > 0)
         {
-            Rice::Object k = elems.shift();
-            rslt[k] = elems.shift();
+            VALUE k = rb_ary_shift(elems);
+            rb_hash_aset(rslt, k, rb_ary_shift(elems));
         }
 
-        o = rslt;
+        v = rslt;
         return p + 1;
     }
     else if (cs == EDN_map_error) {
@@ -729,7 +725,7 @@ const char* edn::Parser::parse_map(const char *p, const char *pe, Rice::Object&
 
     action parse_set {
         // #{ }
-        const char *np = parse_set(fpc, pe, o);
+        const char *np = parse_set(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
@@ -741,7 +737,7 @@ const char* edn::Parser::parse_map(const char *p, const char *pe, Rice::Object&
 
     action parse_tagged {
         // #inst, #uuid, or #user/tag
-        const char *np = parse_tagged(fpc, pe, o);
+        const char *np = parse_tagged(fpc, pe, v);
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
@@ -754,7 +750,7 @@ const char* edn::Parser::parse_map(const char *p, const char *pe, Rice::Object&
 }%%
 
 
-const char* edn::Parser::parse_dispatch(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_dispatch(const char *p, const char *pe, VALUE& v)
 {
     int cs;
 
@@ -794,19 +790,19 @@ const char* edn::Parser::parse_dispatch(const char *p, const char *pe, Rice::Obj
 //
 // set parsing
 //
-const char* edn::Parser::parse_set(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_set(const char *p, const char *pe, VALUE& v)
 {
     static const char* EDN_TYPE = "set";
 
     int cs;
-    Rice::Array elems; // stored as a vector
+    VALUE elems = rb_ary_new(); // stored as an array
 
     %% write init;
     %% write exec;
 
     if (cs >= EDN_set_first_final) {
         // all elements collected; now convert to a set
-        o = Parser::make_ruby_set(elems);
+        v = Parser::make_ruby_set(elems);
         return p + 1;
     }
     else if (cs == EDN_set_error) {
@@ -821,9 +817,8 @@ const char* edn::Parser::parse_set(const char *p, const char *pe, Rice::Object&
 
 // ============================================================
 // discard - consume the discard token and parse the next value to
-// discard. TODO: perhaps optimize this so no object data is built
-// by defining a new machine(s) to consume items within container
-// delimiters
+// discard. TODO: perhaps optimize this so no object data is built by
+// defining a machine to consume items within container delimiters
 //
 %%{
     machine EDN_discard;
@@ -834,12 +829,12 @@ const char* edn::Parser::parse_set(const char *p, const char *pe, Rice::Object&
     begin_discard = '_';
 
     action discard_value {
-        const char *np = parse_value(fpc, pe, o);
+        const char *np = parse_value(fpc, pe, v);
         if (np) {
             // this token is to be discard it so store it in the
             // discard stack - we really don't need to save it so this
             // could be simplified
-            discard.push(o);
+            discard.push(v);
             fexec np;
         } else {
             fhold; fbreak;
@@ -856,7 +851,7 @@ const char* edn::Parser::parse_set(const char *p, const char *pe, Rice::Object&
 const char* edn::Parser::parse_discard(const char *p, const char *pe)
 {
     int cs;
-    Rice::Object o;
+    VALUE v;
 
     %% write init;
     %% write exec;
@@ -914,10 +909,10 @@ const char* edn::Parser::parse_discard(const char *p, const char *pe)
 }%%
 
 
-const char* edn::Parser::parse_tagged(const char *p, const char *pe, Rice::Object& o)
+const char* edn::Parser::parse_tagged(const char *p, const char *pe, VALUE& v)
 {
-    std::string sym_name;
-    Rice::Object data;
+    VALUE sym_name = Qnil;
+    VALUE data = Qnil;
 
     int cs;
 
@@ -930,9 +925,9 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, Rice::Objec
         try {
             // tagged_element makes a call to ruby which may throw an
             // exception when parsing the data
-            o = Parser::tagged_element(sym_name, data);
-        } catch (Rice::Exception& e) {
-            error(__FUNCTION__, e.message().str());
+            v = Parser::tagged_element(sym_name, data);
+        } catch (std::exception& e) {
+            error(__FUNCTION__, e.what());
             return pe;
         }
         return p + 1;
@@ -948,13 +943,14 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, Rice::Objec
 
 
 // ============================================================
-// main parsing machine
+// parses entire input but expects single valid token at the
+// top-level, therefore, does not tokenize source stream
 //
 %%{
-    machine EDN;
+    machine EDN_parser;
     include EDN_common;
 
-    write data nofinal;
+    write data;
 
     action parse_value {
         const char* np = parse_value(fpc, pe, result);
@@ -968,36 +964,71 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, Rice::Objec
     main := ignore* sequence? ignore*;
 }%%
 
-//
-// TODO: Currently using a sequence to handle cases with a discard
-// but EDN's Reader allows token by token parsing
-Rice::Object edn::Parser::parse(const char* buf, std::size_t len)
+
+VALUE edn::Parser::parse(const char* src, std::size_t len)
 {
     int cs;
-    const char *p;
-    const char *pe;
-    Rice::Object result;
+    VALUE result = Qnil;
 
-    // init
-    line_number = 1;
-    p_save = NULL;
-    while (!discard.empty())
-        discard.pop();
+    // reset line counter & discard stack
+    reset();
 
     %% write init;
-    p = &buf[0];
+    p = src;
     pe = p + len;
-    eof = pe; // eof defined in Parser class
+    eof = pe;
     %% write exec;
 
-    if (cs == EDN_error) {
+    if (cs == EDN_parser_error) {
         error(__FUNCTION__, *p);
         return Qnil;
     }
-    else if (cs == EDN_en_main) {} // silence ragel warning
+    else if (cs == EDN_parser_first_final) {
+        // whole source is parsed so reset
+        p = pe = eof = NULL;
+        reset();
+    }
+    else if (cs == EDN_parser_en_main) {} // silence ragel warning
+    return result;
+}
+
+
+// ============================================================
+// token-by-token machine
+//
+%%{
+    machine EDN_tokens;
+    include EDN_common;
+
+    write data noerror nofinal;
+
+    action parse_value {
+        const char* np = parse_value(fpc, pe, result);
+        if (np == NULL) { fhold; fbreak; } else { fexec np; }
+    }
+
+    element       = begin_value >parse_value;
+
+    main := ignore* element ignore*;
+}%%
+
+
+//
+//
+VALUE edn::Parser::next()
+{
+    VALUE result = Qnil;
+    int cs;
+
+    %% write init;
+    %% write exec;
+
+    if (cs == EDN_tokens_en_main) {} // silence ragel warning
+
     return result;
 }
 
+
 /*
  * Local variables:
  * mode: c