edn_turbo 0.5.3 → 0.5.4

data/ext/edn_turbo/edn_parser.rl

@@ -1,12 +1,14 @@
 #include <iostream>
 #include <string>
+#include <sstream>
 #include <vector>
 #include <exception>
 #include <cstring>
 
 #include <ruby/ruby.h>
 
-#include "edn_parser.h"
+#include "util.h"
+#include "parser.h"
 
 //
 // EDN spec at: https://github.com/edn-format/edn
@@ -120,7 +122,7 @@
             else if (std::strcmp(RSTRING_PTR(sym), "false") == 0) { v = Qfalse; }
             else if (std::strcmp(RSTRING_PTR(sym), "nil") == 0)   { v = Qnil; }
             else {
-                v = Parser::make_edn_type(EDNT_MAKE_SYMBOL_METHOD, sym);
+                v = edn::util::call_module_fn(rb_mEDN, EDN_MAKE_SYMBOL_METHOD, sym);
             }
             fexec np;
         }
@@ -207,7 +209,7 @@ const char *edn::Parser::parse_value(const char *p, const char *pe, VALUE& v)
     write data;
 
     action parse_chars {
-        if (Parser::parse_byte_stream(p_save + 1, p, v, encode)) {
+        if (edn::util::parse_byte_stream(p_save + 1, p, v, encode)) {
             fexec p + 1;
         } else {
             fhold; fbreak;
@@ -322,7 +324,7 @@ const char* edn::Parser::parse_decimal(const char *p, const char *pe, VALUE& v)
     %% write exec;
 
     if (cs >= EDN_decimal_first_final) {
-        v = Parser::float_to_ruby(p_save, p - p_save);
+        v = edn::util::float_to_ruby(p_save, p - p_save);
         return p + 1;
     }
     else if (cs == EDN_decimal_en_main) {} // silence ragel warning
@@ -354,7 +356,7 @@ const char* edn::Parser::parse_integer(const char *p, const char *pe, VALUE& v)
     %% write exec;
 
     if (cs >= EDN_integer_first_final) {
-        v = Parser::integer_to_ruby(p_save, p - p_save);
+        v = edn::util::integer_to_ruby(p_save, p - p_save);
         return p + 1;
     }
     else if (cs == EDN_integer_en_main) {} // silence ragel warning
@@ -382,7 +384,7 @@ const char* edn::Parser::parse_integer(const char *p, const char *pe, VALUE& v)
         const char *np = parse_symbol(p_save, pe, sym);
         if (np == NULL) { fexec pe; } else {
             if (sym != Qnil)
-                v = Parser::make_edn_type(EDNT_MAKE_SYMBOL_METHOD, sym);
+                v = edn::util::call_module_fn(rb_mEDN, EDN_MAKE_SYMBOL_METHOD, sym);
             fexec np;
         }
     }
@@ -414,7 +416,7 @@ const char* edn::Parser::parse_integer(const char *p, const char *pe, VALUE& v)
         // stand-alone operators (-, +, /, ... etc)
         char op[2] = { *p_save, 0 };
         VALUE sym = rb_str_new2(op);
-        v = Parser::make_edn_type(EDNT_MAKE_SYMBOL_METHOD, sym);
+        v = edn::util::call_module_fn(rb_mEDN, EDN_MAKE_SYMBOL_METHOD, sym);
     }
 
     valid_non_numeric_chars = alpha|operators|':'|'#';
@@ -479,7 +481,7 @@ const char* edn::Parser::parse_esc_char(const char *p, const char *pe, VALUE& v)
 
     if (cs >= EDN_escaped_char_first_final) {
         // convert the escaped value to a character
-        if (!Parser::parse_escaped_char(p_save + 1, p, v)) {
+        if (!edn::util::parse_escaped_char(p_save + 1, p, v)) {
             return pe;
         }
         return p;
@@ -601,7 +603,7 @@ const char* edn::Parser::parse_symbol(const char *p, const char *pe, VALUE& s)
                     // parse_value() read an element we care
                     // about. Bind the metadata to it and add it to
                     // the sequence
-                    e = Parser::make_edn_type(rb_mEDNT, EDNT_EXTENDED_VALUE_METHOD, e, ruby_meta());
+                    e = edn::util::call_module_fn(rb_mEDNT, EDNT_EXTENDED_VALUE_METHOD, e, ruby_meta());
                     rb_ary_push(elems, e);
                 }
             } else {
@@ -692,7 +694,7 @@ const char* edn::Parser::parse_list(const char *p, const char *pe, VALUE& v)
     if (cs >= EDN_list_first_final) {
         v = elems;
         // TODO: replace with this but first figure out why array is not unrolled by EDN::list()
-        //        v = Parser::make_edn_type(EDNT_MAKE_LIST_METHOD, elems);
+        //        v = edn::util::call_module_fn(EDN_MAKE_LIST_METHOD, elems);
         return p + 1;
     }
     else if (cs == EDN_list_error) {
@@ -853,7 +855,7 @@ const char* edn::Parser::parse_set(const char *p, const char *pe, VALUE& v)
 
     if (cs >= EDN_set_first_final) {
         // all elements collected; now convert to a set
-        v = Parser::make_edn_type(EDNT_MAKE_SET_METHOD, elems);
+        v = edn::util::call_module_fn(rb_mEDN, EDN_MAKE_SET_METHOD, elems);
         return p + 1;
     }
     else if (cs == EDN_set_error) {
@@ -998,14 +1000,14 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, VALUE& v)
 
         if (!sym_ok || !data_ok) {
             error(__FUNCTION__, "tagged element symbol error", *p);
-            v =  EDNT_EOF_CONST;
+            v = EDN_EOF_CONST;
             return NULL;
         }
 
         try {
             // tagged_element makes a call to ruby which may throw an
             // exception when parsing the data
-            v = Parser::make_edn_type(EDNT_TAGGED_ELEM_METHOD, sym_name, data);
+            v = edn::util::call_module_fn(rb_mEDN, EDN_TAGGED_ELEM_METHOD, sym_name, data);
             return p + 1;
         } catch (std::exception& e) {
             error(__FUNCTION__, e.what());
@@ -1016,7 +1018,7 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, VALUE& v)
         error(__FUNCTION__, "tagged element symbol error", *p);
     }
     else if (cs == EDN_tagged_en_main) {} // silence ragel warning
-    v =  EDNT_EOF_CONST;
+    v =  EDN_EOF_CONST;
     return NULL;
 }
 
@@ -1090,7 +1092,7 @@ const char* edn::Parser::parse_meta(const char *p, const char *pe)
             // metadata sequence to it
             if (!meta_empty() && meta_size() == meta_sz) {
                 // this will empty the metadata sequence too
-                result = Parser::make_edn_type(rb_mEDNT, EDNT_EXTENDED_VALUE_METHOD, result, ruby_meta());
+                result = edn::util::call_module_fn(rb_mEDNT, EDNT_EXTENDED_VALUE_METHOD, result, ruby_meta());
             }
             fexec np;
         }
@@ -1107,7 +1109,7 @@ const char* edn::Parser::parse_meta(const char *p, const char *pe)
 VALUE edn::Parser::parse(const char* src, std::size_t len)
 {
     int cs;
-    VALUE result = EDNT_EOF_CONST;
+    VALUE result = EDN_EOF_CONST;
 
     %% write init;
     set_source(src, len);
@@ -1115,7 +1117,7 @@ VALUE edn::Parser::parse(const char* src, std::size_t len)
 
     if (cs == EDN_parser_error) {
         error(__FUNCTION__, *p);
-        return EDNT_EOF_CONST;
+        return EDN_EOF_CONST;
     }
     else if (cs == EDN_parser_first_final) {
         p = pe = eof = NULL;
@@ -1152,7 +1154,7 @@ VALUE edn::Parser::parse(const char* src, std::size_t len)
                 else {
                     // a value was read and there's a pending metadata
                     // sequence. Bind them.
-                    value = Parser::make_edn_type(rb_mEDNT, EDNT_EXTENDED_VALUE_METHOD, value, ruby_meta());
+                    value = edn::util::call_module_fn(rb_mEDNT, EDNT_EXTENDED_VALUE_METHOD, value, ruby_meta());
                     state = TOKEN_OK;
                 }
             } else if (!discard.empty()) {

data/ext/edn_turbo/extconf.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'mkmf'
 
 HEADER_DIRS = [
@@ -13,17 +15,11 @@ LIB_DIRS = [
 
 dir_config('edn_ext', HEADER_DIRS, LIB_DIRS)
 
-# feels very hackish to do this
-if RUBY_PLATFORM =~ /darwin/
-  $CXXFLAGS << ' -stdlib=libc++ -std=c++11'
-end
+# feels very hackish to do this but the new icu4c needs it on MacOS
+$CXXFLAGS << ' -stdlib=libc++ -std=c++11' if RUBY_PLATFORM.match?(/darwin/)
 
-unless find_header('unicode/uversion.h')
-  abort "icu4c headers missing"
-end
+abort 'icu4c headers missing' unless find_header('unicode/uversion.h')
 
-unless have_library('icuuc')
-  abort "ic4c lib missing"
-end
+abort 'ic4c lib missing' unless have_library('icuuc')
 
-create_makefile("edn_turbo/edn_turbo")
+create_makefile('edn_turbo/edn_turbo')

data/ext/edn_turbo/main.cc

@@ -6,28 +6,27 @@
 #include <ruby/ruby.h>
 #include <ruby/io.h>
 
-#include "edn_parser.h"
+#include "parser.h"
 
 namespace edn {
 
+    VALUE rb_mEDN;
     VALUE rb_mEDNT;
 
     // Symbols used to call into the ruby world.
-    VALUE EDN_MODULE_SYMBOL             = Qnil;
+    VALUE EDN_MAKE_SYMBOL_METHOD       = Qnil;
+    VALUE EDN_MAKE_LIST_METHOD         = Qnil;
+    VALUE EDN_MAKE_SET_METHOD          = Qnil;
+    VALUE EDN_MAKE_BIG_DECIMAL_METHOD  = Qnil;
+    VALUE EDN_TAGGED_ELEM_METHOD       = Qnil;
+    VALUE EDNT_EXTENDED_VALUE_METHOD   = Qnil;
 
-    VALUE EDNT_MAKE_SYMBOL_METHOD       = Qnil;
-    VALUE EDNT_MAKE_LIST_METHOD         = Qnil;
-    VALUE EDNT_MAKE_SET_METHOD          = Qnil;
-    VALUE EDNT_MAKE_BIG_DECIMAL_METHOD  = Qnil;
-    VALUE EDNT_TAGGED_ELEM_METHOD       = Qnil;
-    VALUE EDNT_EXTENDED_VALUE_METHOD    = Qnil;
-
-    VALUE EDNT_STRING_TO_I_METHOD       = Qnil;
-    VALUE EDNT_STRING_TO_F_METHOD       = Qnil;
-    VALUE EDNT_READ_METHOD              = Qnil;
+    VALUE RUBY_STRING_TO_I_METHOD      = Qnil;
+    VALUE RUBY_STRING_TO_F_METHOD      = Qnil;
+    VALUE RUBY_READ_METHOD             = Qnil;
 
     // returned when EOF - defined as a constant in EDN module
-    VALUE EDNT_EOF_CONST                = Qnil;
+    VALUE EDN_EOF_CONST                = Qnil;
 
     //
     // Wrappers to hook the class w/ the C-api.
@@ -108,7 +107,7 @@ namespace edn {
               // this is very inefficient as it'll require read()
               // calls from the ruby side (involves a lot of data
               // wrapping, etc)
-              if (rb_respond_to(data, EDNT_READ_METHOD)) {
+              if (rb_respond_to(data, RUBY_READ_METHOD)) {
                   p->set_source(data);
                   break;
               }
@@ -174,6 +173,7 @@ void Init_edn_turbo(void)
         rb_raise(rb_eRuntimeError, "Extension init error calling setlocale() - It appears your system's locale is not configured correctly.\n");
     }
 
+    edn::rb_mEDN  = rb_const_get(rb_cObject, rb_intern("EDN"));
     edn::rb_mEDNT = rb_define_module("EDNT");
 
     // bind the ruby Parser class to the C++ one
@@ -187,19 +187,20 @@ void Init_edn_turbo(void)
     rb_define_method(rb_cParser, "read", (VALUE(*)(ANYARGS)) &edn::next, 0 );
 
     // bind ruby methods we'll call - these should be defined in edn_turbo.rb
-    edn::EDN_MODULE_SYMBOL             = rb_intern("EDN");
-    edn::EDNT_MAKE_SYMBOL_METHOD       = rb_intern("symbol");
-    edn::EDNT_MAKE_LIST_METHOD         = rb_intern("list");
-    edn::EDNT_MAKE_SET_METHOD          = rb_intern("set");
-    edn::EDNT_MAKE_BIG_DECIMAL_METHOD  = rb_intern("big_decimal");
-    edn::EDNT_TAGGED_ELEM_METHOD       = rb_intern("tagged_element");
-    edn::EDNT_EXTENDED_VALUE_METHOD    = rb_intern("extend_for_meta");
-
-    edn::EDNT_STRING_TO_I_METHOD       = rb_intern("to_i");
-    edn::EDNT_STRING_TO_F_METHOD       = rb_intern("to_f");
-    edn::EDNT_READ_METHOD              = rb_intern("read");
+    edn::EDN_MAKE_SYMBOL_METHOD        = rb_intern("symbol");
+    edn::EDN_MAKE_LIST_METHOD          = rb_intern("list");
+    edn::EDN_MAKE_SET_METHOD           = rb_intern("set");
+    edn::EDN_MAKE_BIG_DECIMAL_METHOD   = rb_intern("big_decimal");
+    edn::EDN_TAGGED_ELEM_METHOD        = rb_intern("tagged_element");
+
+    // defined in EDNT - see edn_parser.rb
+    edn::EDNT_EXTENDED_VALUE_METHOD     = rb_intern("extend_for_meta");
+
+    // ruby methods
+    edn::RUBY_STRING_TO_I_METHOD       = rb_intern("to_i");
+    edn::RUBY_STRING_TO_F_METHOD       = rb_intern("to_f");
+    edn::RUBY_READ_METHOD              = rb_intern("read");
 
     // so we can return EOF directly
-    VALUE edn_module                   = rb_const_get(rb_cObject, edn::EDN_MODULE_SYMBOL);
-    edn::EDNT_EOF_CONST                = rb_const_get(edn_module, rb_intern("EOF"));
+    edn::EDN_EOF_CONST                 = rb_const_get(edn::rb_mEDN, rb_intern("EOF"));
 }

data/ext/edn_turbo/{edn_parser.h → parser.h}

@@ -1,29 +1,12 @@
 #pragma once
 
 #include <string>
-#include <sstream>
 #include <vector>
 #include <stack>
 
 
 namespace edn
 {
-    extern VALUE rb_mEDNT;
-    extern VALUE EDN_MODULE_SYMBOL;
-    extern VALUE EDNT_MAKE_SYMBOL_METHOD;
-    extern VALUE EDNT_MAKE_LIST_METHOD;
-    extern VALUE EDNT_MAKE_SET_METHOD;
-    extern VALUE EDNT_MAKE_BIG_DECIMAL_METHOD;
-    extern VALUE EDNT_TAGGED_ELEM_METHOD;
-    extern VALUE EDNT_EXTENDED_VALUE_METHOD;
-
-    extern VALUE EDNT_STRING_TO_I_METHOD;
-    extern VALUE EDNT_STRING_TO_F_METHOD;
-    extern VALUE EDNT_READ_METHOD;
-
-    extern VALUE EDNT_EOF_CONST;
-
-
     //
     // C-extension EDN Parser class representation
     class Parser
@@ -50,8 +33,6 @@ namespace edn
         // returns the next element in the current stream
         VALUE next();
 
-        static void throw_error(int error);
-
     private:
         // ragel needs these
         const char* p;
@@ -89,18 +70,6 @@ namespace edn
 
         eTokenState parse_next(VALUE& value);
 
-        // defined in edn_parser_util.cc
-        static VALUE integer_to_ruby(const char* str, std::size_t len);
-        static VALUE float_to_ruby  (const char* str, std::size_t len);
-        static VALUE ruby_io_read(VALUE io);
-
-        static bool parse_byte_stream (const char *p, const char *pe, VALUE& rslt, bool encode);
-        static bool parse_escaped_char(const char *p, const char *pe, VALUE& rslt);
-
-        static VALUE make_edn_type(ID method, VALUE value);
-        static VALUE make_edn_type(ID method, VALUE value1, VALUE value2);
-        static VALUE make_edn_type(VALUE module, ID method, VALUE value1, VALUE value2);
-
         // metadata
         VALUE ruby_meta();
         void  new_meta_list() { metadata.push( new std::vector<VALUE>() ); }
@@ -109,17 +78,6 @@ namespace edn
         bool  meta_empty() const { return metadata.top()->empty(); }
         std::size_t meta_size() const { return metadata.top()->size(); }
 
-        // utility method to convert a primitive in string form to a
-        // ruby type
-        template <class T>
-        static inline T buftotype(const char* p, std::size_t len) {
-            T val;
-            std::string buf;
-            buf.append(p, len);
-            std::istringstream(buf) >> val;
-            return val;
-        }
-
         void error(const std::string& f, const std::string& err, char c) const;
         void error(const std::string& f, char err_c) const { error(f, "", err_c); }
         void error(const std::string& f, const std::string& err_msg) const { error(f, err_msg, '\0'); }

data/ext/edn_turbo/parser_def.cc

@@ -0,0 +1,197 @@
+#include <iostream>
+#include <string>
+#include <stack>
+
+#include <ruby/ruby.h>
+
+#include "parser.h"
+#include "util.h"
+
+namespace edn
+{
+    //
+    // parser destructor
+    Parser::~Parser()
+    {
+        reset_state();
+        del_top_meta_list();
+
+        if (io_buffer) {
+            free(reinterpret_cast<void*>(io_buffer));
+        }
+    }
+    
+    // =================================================================
+    // for token-by-token parsing. If a discard or metadata is parsed,
+    // attempt to get the following value
+    //
+    VALUE Parser::next()
+    {
+        VALUE token = EDN_EOF_CONST;
+
+        // buffer if reading from an IO
+        if (core_io || (read_io != Qnil)) {
+            fill_buf();
+        }
+
+        while (!is_eof())
+        {
+            // fetch a token. If it's metadata or discard
+            VALUE v = EDN_EOF_CONST;
+            eTokenState state = parse_next(v);
+
+            if (state == TOKEN_OK) {
+                // valid token
+                token = v;
+                break;
+            }
+            else if (state == TOKEN_ERROR) {
+                token = EDN_EOF_CONST;
+                break;
+            }
+        }
+
+        return token;
+    }
+
+    // reset parsing state
+    //
+    void Parser::reset_state()
+    {
+        line_number = 1;
+        discard.clear();
+
+        // remove any remaining levels except for the first
+        while (metadata.size() > 1) {
+            del_top_meta_list();
+        }
+        // but clear any metadata on the first
+        metadata.top()->clear();
+
+        // clean up
+        core_io = NULL;
+        read_io = Qnil;
+        p = pe = eof = NULL;
+    }
+
+    //
+    // set a new source
+    void Parser::set_source(const char* src, std::size_t len)
+    {
+        reset_state();
+        // set ragel state
+        p = src;
+        pe = src + len;
+        eof = pe;
+    }
+
+    void Parser::set_source(FILE* fp)
+    {
+        reset_state();
+        core_io = fp;
+    }
+
+    void Parser::set_source(VALUE str_io)
+    {
+        reset_state();
+        read_io = str_io;
+    }
+
+    //
+    // for IO sources, read and fill a buffer
+    void Parser::fill_buf()
+    {
+        std::string str_buf;
+
+        // read as much data available
+        if (core_io) {
+            // ruby core IO types
+            char c;
+            while (1)
+            {
+                c = fgetc(core_io);
+                if (c == EOF) {
+                    break;
+                }
+                str_buf += c;
+            }
+
+        } else if (read_io != Qnil) {
+            // StringIO, etc. Call read() from ruby side
+            VALUE v = edn::util::ruby_io_read(read_io);
+            if (TYPE(v) == T_STRING) {
+                str_buf.assign( StringValuePtr(v), RSTRING_LEN(v));
+            }
+        }
+
+        // set the buffer to read from
+        if (str_buf.length() > 0) {
+            // first time when io_buffer is NULL, pe & p = 0
+            uintmax_t new_length = (pe - p) + str_buf.length();
+            if (new_length > (((uintmax_t) 1 << 32) - 1)) {
+                // icu -> 32-bit. TODO: handle
+                rb_raise(rb_eRuntimeError, "Unsupported string buffer length");
+            }
+            char* start = NULL;
+
+            // allocate or extend storage needed
+            if (!io_buffer) {
+                io_buffer = reinterpret_cast<char*>(malloc(new_length));
+                start = io_buffer;
+            } else if (io_buffer_len < new_length) {
+                // resize the buffer
+                io_buffer = reinterpret_cast<char*>(realloc(reinterpret_cast<void*>(io_buffer), new_length));
+            }
+
+            if (!start) {
+                // appending to the buffer but move the data not yet
+                // parsed first to the front
+                memmove(io_buffer, p, pe - p);
+                start = io_buffer + (pe - p);
+            }
+            
+            // and copy
+            memcpy(start, str_buf.c_str(), str_buf.length()); 
+            io_buffer_len = (uint32_t) new_length;
+
+            // set ragel state
+            p = io_buffer;
+            pe = p + new_length;
+            eof = pe;
+        }
+    }
+
+    // =================================================================
+    // METADATA
+    //
+    // returns an array of metadata value(s) saved in reverse order
+    // (right to left) - the ruby side will interpret this
+    VALUE Parser::ruby_meta()
+    {
+        VALUE m_ary = rb_ary_new();
+
+        // pop from the back of the top-most list
+        while (!metadata.top()->empty()) {
+            rb_ary_push(m_ary, metadata.top()->back());
+            metadata.top()->pop_back();
+        }
+
+        return m_ary;
+    }
+
+
+    // =================================================================
+    //
+    // error reporting
+    void Parser::error(const std::string& func, const std::string& err, char c) const
+    {
+        std::cerr << "Parse error "
+            //            "from " << func << "() "
+            ;
+        if (err.length() > 0)
+            std::cerr << "(" << err << ") ";
+        if (c != '\0')
+            std::cerr << "at '" << c << "' ";
+        std::cerr << "on line " << line_number << std::endl;
+    }
+}