edn_turbo 0.2.2 → 0.3.0

data/ext/edn_turbo/edn_parser_util.cc

@@ -2,9 +2,7 @@
 #include <iomanip>
 #include <string>
 #include <limits>
-
-#include <rice/String.hpp>
-#include <rice/Array.hpp>
+#include <exception>
 
 #include <ruby/ruby.h>
 #include <ruby/encoding.h>
@@ -23,43 +21,121 @@ namespace edn
         return s.str().length();
     }
 
-    static const std::size_t LL_max_chars = get_max_chars<>((long long) 1);
-    static const std::size_t LD_max_chars = get_max_chars<>((long double) 1);
+    static const std::size_t LL_max_chars = get_max_chars<>((long) 1);
+    static const std::size_t LD_max_chars = get_max_chars<>((double) 1);
+
+
+    // =================================================================
+    // reset parsing state
+    //
+    void Parser::reset()
+    {
+        line_number = 1;
+        while (!discard.empty())
+            discard.pop();
+    }
+
+    //
+    // set a new source
+    void Parser::set_source(const char* src, std::size_t len)
+    {
+        reset();
+        // set ragel state
+        p = src;
+        pe = src + len;
+        eof = pe;
+    }
+
+
+    // =================================================================
+    // work-around for idiotic rb_protect convention in order to avoid
+    // using ruby/rice
+    //
+    typedef VALUE (edn_rb_f_type)( VALUE arg );
+
+    // we're using at most 2 args
+    struct prot_args {
+        prot_args(ID m, VALUE arg) :
+            method(m), count(1) {
+            args[0] = arg;
+        }
+        prot_args(ID m, VALUE arg1, VALUE arg2) :
+            method(m), count(2) {
+            args[0] = arg1;
+            args[1] = arg2;
+        }
+
+        ID method;
+        VALUE count;
+        VALUE args[2];
+    };
+
+    // this allows us to wrap with rb_protect()
+    static inline VALUE edn_wrap_funcall2( VALUE arg )
+    {
+        prot_args* a = reinterpret_cast<prot_args*>(arg);
+        return rb_funcall2( edn::rb_mEDNT, a->method, a->count, a->args );
+    }
+
+    static inline VALUE edn_prot_rb_funcall( edn_rb_f_type func, VALUE args )
+    {
+        int error;
+        VALUE s = rb_protect( func, args, &error );
+        if (error) Parser::throw_error(error);
+        return s;
+    }
+
+    static inline VALUE edn_prot_rb_new_str(const char* str) {
+        int error;
+        VALUE s = rb_protect( reinterpret_cast<VALUE (*)(VALUE)>(rb_str_new_cstr),
+                              reinterpret_cast<VALUE>(str), &error );
+        if (error) Parser::throw_error(error);
+        return s;
+    }
+
+    static inline VALUE edn_rb_enc_associate_utf8(VALUE str)
+    {
+        return rb_enc_associate(str, rb_utf8_encoding() );
+    }
+
+    // =================================================================
+    // utils
 
     //
     // convert to int.. if string rep has more digits than long can
     // hold, call into ruby to get a big num
-    Rice::Object Parser::integer_to_ruby(const char* str, std::size_t len)
+    VALUE Parser::integer_to_ruby(const char* str, std::size_t len)
     {
         if (len < LL_max_chars)
         {
-            return buftotype<long>(str, len);
+            return LONG2NUM(buftotype<long>(str, len));
         }
 
         // value is outside of range of long type. Use ruby to convert it
-        VALUE rb_s = Rice::protect(rb_str_new2, str);
-        return Rice::protect(rb_funcall, rb_mEDNT, EDNT_STR_INT_TO_BIGNUM, 1, rb_s);
+        VALUE rb_s = edn_prot_rb_new_str( str );
+        prot_args args(EDNT_STR_INT_TO_BIGNUM, rb_s);
+        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
     //
-    // as above.. TODO: check exponential
-    Rice::Object Parser::float_to_ruby(const char* str, std::size_t len)
+    // as above.. TODO: check exponential..
+    VALUE Parser::float_to_ruby(const char* str, std::size_t len)
     {
         if (len < LD_max_chars)
         {
-            return buftotype<double>(str, len);
+            return rb_float_new(buftotype<double>(str, len));
         }
 
         // value is outside of range of long type. Use ruby to convert it
-        VALUE rb_s = Rice::protect(rb_str_new2, str);
-        return Rice::protect(rb_funcall, rb_mEDNT, EDNT_STR_DBL_TO_BIGNUM, 1, rb_s);
+        prot_args args(EDNT_STR_DBL_TO_BIGNUM, edn_prot_rb_new_str(str));
+        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
 
     //
     // copies the string data, unescaping any present values that need to be replaced
     //
-    bool Parser::parse_byte_stream(const char *p_start, const char *p_end, Rice::String& s,
+    bool Parser::parse_byte_stream(const char *p_start, const char *p_end, VALUE& v_utf8,
                                    bool encode)
     {
         if (p_end > p_start) {
@@ -74,9 +150,13 @@ namespace edn
             }
 
             // utf-8 encode
-            VALUE vs = Rice::protect( rb_str_new2, buf.c_str() );
-            VALUE s_utf8 = Rice::protect( rb_enc_associate, vs, rb_utf8_encoding() );
-            s = Rice::String(s_utf8);
+            VALUE vs = edn_prot_rb_new_str(buf.c_str());
+            int error;
+            v_utf8 = rb_protect( edn_rb_enc_associate_utf8, vs, &error);
+            if (error) Parser::throw_error(error);
+            return true;
+        } else if (p_end == p_start) {
+            v_utf8 = rb_str_new("", 0);
             return true;
         }
 
@@ -86,7 +166,7 @@ namespace edn
     //
     // handles things like \c, \newline
     //
-    bool Parser::parse_escaped_char(const char *p, const char *pe, Rice::Object& o)
+    bool Parser::parse_escaped_char(const char *p, const char *pe, VALUE& v)
     {
         std::string buf;
         std::size_t len = pe - p;
@@ -104,37 +184,51 @@ namespace edn
             else return false;
         }
 
-        o = Rice::String(buf);
+        v = edn_prot_rb_new_str( buf.c_str() );
         return true;
     }
 
 
     //
     // get a set representation from the ruby side. See edn_turbo.rb
-    Rice::Object Parser::make_edn_symbol(const std::string& name)
+    VALUE Parser::make_edn_symbol(VALUE sym)
     {
-        VALUE rb_s = Rice::protect(rb_str_new2, name.c_str());
-        return Rice::protect(rb_funcall, rb_mEDNT, EDNT_MAKE_EDN_SYMBOL, 1, rb_s);
+        prot_args args(edn::EDNT_MAKE_EDN_SYMBOL, sym);
+        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
     //
     // get a set representation from the ruby side. See edn_turbo.rb
-    Rice::Object Parser::make_ruby_set(const Rice::Array& elems)
+    VALUE Parser::make_ruby_set(VALUE elems)
     {
-        return Rice::protect(rb_funcall, rb_mEDNT, EDNT_MAKE_SET_METHOD, 1, elems.value());
+        prot_args args(edn::EDNT_MAKE_SET_METHOD, elems);
+        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
     //
     // get an object representation from the ruby side using the given symbol name
-    Rice::Object Parser::tagged_element(const std::string& name, const Rice::Object& data)
+    VALUE Parser::tagged_element(VALUE name, VALUE data)
     {
-        VALUE rb_s = Rice::protect(rb_str_new2, name.c_str());
-        return Rice::protect(rb_funcall, rb_mEDNT, EDNT_TAGGED_ELEM, 2, rb_s, data.value());
+        prot_args args(edn::EDNT_TAGGED_ELEM, name, data);
+        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
 
     //
     // error reporting
+    void Parser::throw_error(int error)
+    {
+        if (error == 0)
+            return;
+
+        VALUE err = rb_errinfo();
+        VALUE klass = rb_class_path(CLASS_OF(err));
+        VALUE message = rb_obj_as_string(err);
+        std::stringstream msg;
+        msg << RSTRING_PTR(klass) << " exception: " << RSTRING_PTR(message);
+        throw std::runtime_error(msg.str());
+    }
+
     void Parser::error(const std::string& func, const std::string& err, char c) const
     {
         std::cerr << "Parse error "

data/ext/edn_turbo/extconf.rb

@@ -1,4 +1,4 @@
-require 'mkmf-rice'
+require 'mkmf'
 
 HEADER_DIRS = [
   '/usr/local/include',

data/ext/edn_turbo/main.cc

@@ -2,16 +2,14 @@
 #include <iostream>
 #include <clocale>
 
-// always include rice headers before ruby.h
-#include <rice/Data_Type.hpp>
-#include <rice/Constructor.hpp>
-
 #include "edn_parser.h"
 
+#include <ruby/ruby.h>
+
 
 namespace edn {
 
-    Rice::Module rb_mEDNT;
+    VALUE rb_mEDNT;
 
     // methods on the ruby side we'll call from here
     VALUE EDNT_MAKE_EDN_SYMBOL = Qnil;
@@ -20,8 +18,83 @@ namespace edn {
     VALUE EDNT_STR_INT_TO_BIGNUM = Qnil;
     VALUE EDNT_STR_DBL_TO_BIGNUM = Qnil;
 
+    //
+    // wrappers to hook the class w/ the C-api
+    template<class T>
+    static void delete_obj(T *ptr) {
+        delete ptr;
+    }
+
+    template<class T>
+    static VALUE wrap_ptr(VALUE klass, T* ptr) {
+        return Data_Wrap_Struct(klass, 0, delete_obj<T>, ptr);
+    }
 
-    void die(int sig)
+    static VALUE alloc_obj(VALUE self){
+        return wrap_ptr<edn::Parser>(self, new Parser());
+    }
+
+    static inline Parser* get_parser(VALUE self)
+    {
+        Parser *p;
+        Data_Get_Struct( self, edn::Parser, p );
+        return p;
+    }
+
+
+    //
+    // called by the constructor - sets the source if passed
+    static VALUE initialize(int argc, VALUE* argv, VALUE self)
+    {
+        Parser* p = get_parser(self);
+
+        if (argc > 0)
+        {
+            const char* stream = StringValueCStr(argv[0]);
+            if (stream)
+                p->set_source( stream, std::strlen(stream) );
+        }
+        return self;
+    }
+
+    //
+    // change the input source
+    static VALUE set_source(VALUE self, VALUE data)
+    {
+        Parser* p = get_parser(self);
+
+        const char* stream = StringValueCStr(data);
+        if (stream)
+            p->set_source( stream, std::strlen(stream) );
+
+        return self;
+    }
+
+    //
+    // eof?
+    static VALUE eof(VALUE self, VALUE data)
+    {
+        return get_parser(self)->is_eof();
+    }
+
+    //
+    // parses an entire stream
+    static VALUE read(VALUE self, VALUE data)
+    {
+        const char* stream = StringValueCStr(data);
+        return get_parser(self)->parse(stream, std::strlen(stream) );
+    }
+
+    //
+    // gets the next token in the current stream
+    static VALUE next(VALUE self, VALUE data)
+    {
+        return get_parser(self)->next();
+    }
+
+    //
+    // signal handler
+    static void die(int sig)
     {
         exit(-1);
     }
@@ -45,22 +118,24 @@ void Init_edn_turbo(void)
         return;
     }
 
-    edn::rb_mEDNT = Rice::define_module("EDNT");
+    edn::rb_mEDNT = rb_define_module("EDNT");
 
-    // bind methods we'll call - these should be defined in edn_turbo.rb
+    // bind the ruby Parser class to the C++ one
+    VALUE rb_cParser = rb_define_class_under(edn::rb_mEDNT, "Parser", rb_cObject);
+    rb_define_alloc_func(rb_cParser, edn::alloc_obj);
+    rb_define_method(rb_cParser, "initialize", (VALUE(*)(ANYARGS)) &edn::initialize, -1 );
+    rb_define_method(rb_cParser, "ext_set_stream", (VALUE(*)(ANYARGS)) &edn::set_source, 1 );
+    rb_define_method(rb_cParser, "ext_eof", (VALUE(*)(ANYARGS)) &edn::eof, 0 );
+    rb_define_method(rb_cParser, "ext_read", (VALUE(*)(ANYARGS)) &edn::read, 1 );
+    rb_define_method(rb_cParser, "ext_next", (VALUE(*)(ANYARGS)) &edn::next, 0 );
+
+    // bind ruby methods we'll call - these should be defined in edn_turbo.rb
     edn::EDNT_MAKE_EDN_SYMBOL = rb_intern("make_edn_symbol");
     edn::EDNT_MAKE_SET_METHOD = rb_intern("make_set");
     edn::EDNT_TAGGED_ELEM = rb_intern("tagged_element");
     edn::EDNT_STR_INT_TO_BIGNUM = rb_intern("string_int_to_bignum");
     edn::EDNT_STR_DBL_TO_BIGNUM = rb_intern("string_double_to_bignum");
 
-    // bind the ruby Parser class to the C++ one
-    Rice::Data_Type<edn::Parser> rb_cParser =
-        Rice::define_class_under<edn::Parser>(edn::rb_mEDNT, "Parser")
-        .define_constructor(Rice::Constructor<edn::Parser>())
-        .define_method("ext_read", &edn::Parser::process, (Rice::Arg("data")))
-        ;
-
     // import whatever else we've defined in the ruby side
     rb_require("edn_turbo/edn_parser");
 }

data/lib/edn_turbo.rb

@@ -16,7 +16,7 @@ module EDNT
     begin
       data = input.instance_of?(String) ? input : input.read
 
-      Parser.new.read(data)
+      Parser.new.parse(data)
     rescue EOFError
     end
 

data/lib/edn_turbo/edn_parser.rb

@@ -1,10 +1,27 @@
 module EDNT
 
+  EOF = Object.new
+
   class Parser
-    def read(data)
-      # call the c-side method
+
+    # initialize() is defined in the c-side (main.cc)
+
+    # call the c-side method
+    def set_input(data)
+      ext_set_stream(data)
+    end
+
+    # token-by-token read
+    def read
+      return EOF if ext_eof
+      ext_next
+    end
+
+    # entire stream read
+    def parse(data)
       ext_read(data)
     end
+
   end
 
 end

data/lib/edn_turbo/version.rb

@@ -1,4 +1,4 @@
 module EDNT
-  VERSION = '0.2.2'
-  RELEASE_DATE = %q{2015-06-03}
+  VERSION = '0.3.0'
+  RELEASE_DATE = %q{2015-06-12}
 end

data/test/test_output_diff.rb

@@ -13,17 +13,17 @@ class EDNT_Test < Minitest::Test
 
   def check_file(file, expected_output)
     File.open(file) { |file|
-      assert_equal(expected_output, @parser.read(file.read))
+      assert_equal(expected_output, @parser.parse(file.read))
     }
   end
 
   def test_basic
 
-    assert_equal(false, @parser.read('false'))
-    assert_equal(true, @parser.read('true'))
-    assert_equal("a string", @parser.read('"a string"'))
-    assert_equal(:"namespace.of.some.length/keyword-name", @parser.read(':namespace.of.some.length/keyword-name'))
-    assert_equal(:'/', @parser.read(':/'))
+    assert_equal(false, @parser.parse('false'))
+    assert_equal(true, @parser.parse('true'))
+    assert_equal("a string", @parser.parse('"a string"'))
+    assert_equal(:"namespace.of.some.length/keyword-name", @parser.parse(':namespace.of.some.length/keyword-name'))
+    assert_equal(:'/', @parser.parse(':/'))
   end
 
   def test_number
@@ -103,7 +103,7 @@ class EDNT_Test < Minitest::Test
   def test_read
 
     # check read for using string
-    assert_equal({:a=>1, :b=>2}, @parser.read('{:a 1 :b 2}'))
+    assert_equal({:a=>1, :b=>2}, @parser.parse('{:a 1 :b 2}'))
 
   end
 
@@ -148,7 +148,7 @@ class EDNT_Test < Minitest::Test
       Tagged.new(data).to_s
     end
 
-    assert_equal([345, :a], @parser.read('#edn_turbo/test_tagged { :item 345 :other :a }'))
+    assert_equal([345, :a], @parser.parse('#edn_turbo/test_tagged { :item 345 :other :a }'))
   end
 
   def test_symbols