edn_turbo 0.4.1 → 0.5.0

data/ext/edn_turbo/edn_parser.h

@@ -1,13 +1,10 @@
-#ifndef EDN_RAGEL_PARSER_H
-#define EDN_RAGEL_PARSER_H
+#pragma once
 
 #include <string>
 #include <sstream>
 #include <vector>
 #include <stack>
 
-#include <ruby/ruby.h>
-
 
 namespace edn
 {
@@ -22,6 +19,7 @@ namespace edn
 
     extern VALUE EDNT_STRING_TO_I_METHOD;
     extern VALUE EDNT_STRING_TO_F_METHOD;
+    extern VALUE EDNT_READ_METHOD;
 
     extern VALUE EDNT_EOF_CONST;
 
@@ -31,13 +29,18 @@ namespace edn
     class Parser
     {
     public:
-        Parser() : p(NULL), pe(NULL), eof(NULL), line_number(1) {
+        Parser() : p(NULL), pe(NULL), eof(NULL),
+          core_io(NULL), read_io(Qnil),
+          io_buffer(NULL), io_buffer_len(0),
+          line_number(1) {
             new_meta_list();
         }
-        ~Parser() { reset_state(); del_top_meta_list(); }
+        ~Parser();
 
         // change input source
         void set_source(const char* src, std::size_t len);
+        void set_source(FILE* fp);
+        void set_source(VALUE string_io);
 
         bool is_eof() const { return (p == eof); }
 
@@ -54,11 +57,16 @@ namespace edn
         const char* p;
         const char* pe;
         const char* eof;
+        FILE* core_io;  // for IO streams
+        VALUE read_io;  // for non-core IO that responds to read()
+        char* io_buffer;
+        uintmax_t io_buffer_len;
         std::size_t line_number;
         std::vector<VALUE> discard;
         std::stack<std::vector<VALUE>* > metadata;
 
         void reset_state();
+        void fill_buf();
 
         const char* parse_value   (const char *p, const char *pe, VALUE& v);
         const char* parse_string  (const char *p, const char *pe, VALUE& v);
@@ -81,12 +89,10 @@ namespace edn
 
         eTokenState parse_next(VALUE& value);
 
-        // defined in edn_parser_unicode.cc
-        static bool to_utf8(const char *s, std::size_t len, std::string& rslt);
-
         // defined in edn_parser_util.cc
         static VALUE integer_to_ruby(const char* str, std::size_t len);
         static VALUE float_to_ruby  (const char* str, std::size_t len);
+        static VALUE ruby_io_read(VALUE io);
 
         static bool parse_byte_stream (const char *p, const char *pe, VALUE& rslt, bool encode);
         static bool parse_escaped_char(const char *p, const char *pe, VALUE& rslt);
@@ -117,8 +123,6 @@ namespace edn
         void error(const std::string& f, const std::string& err, char c) const;
         void error(const std::string& f, char err_c) const { error(f, "", err_c); }
         void error(const std::string& f, const std::string& err_msg) const { error(f, err_msg, '\0'); }
-    }; // Engine
+    }; // Parser
 
 } // namespace
-
-#endif

data/ext/edn_turbo/edn_parser.rl

@@ -2,9 +2,10 @@
 #include <string>
 #include <vector>
 #include <exception>
-
 #include <cstring>
 
+#include <ruby/ruby.h>
+
 #include "edn_parser.h"
 
 //
@@ -222,7 +223,7 @@ const char *edn::Parser::parse_value(const char *p, const char *pe, VALUE& v)
                            ((0xc2..0xf5) |
                             '\\'[\"\\/bfnrt] |
                             '\\u'[0-9a-fA-F]{4}) $mark_for_encoding |
-                           '\\'^([\"\\/bfnrtu]))* %parse_chars
+                            '\\'^([\"\\/bfnrtu]))* %parse_chars
                           ) :>> string_delim @err(close_err) @exit;
 }%%
 
@@ -1189,11 +1190,9 @@ edn::Parser::eTokenState edn::Parser::parse_next(VALUE& value)
     return state;
 }
 
-
 /*
- * Local variables:
- * mode: c
- * c-file-style: ruby
- * indent-tabs-mode: nil
- * End:
- */
+- * Local variables:
+- * mode: c
+- * indent-tabs-mode: nil
+- * End:
+- */

data/ext/edn_turbo/edn_parser_util.cc

@@ -12,6 +12,7 @@
 #include <ruby/encoding.h>
 
 #include "edn_parser.h"
+#include "edn_parser_util.h"
 
 namespace edn
 {
@@ -29,6 +30,18 @@ namespace edn
     static const std::size_t LD_max_chars = get_max_chars<>((double) 1);
 
 
+    // parser destructor
+    //
+    Parser::~Parser()
+    {
+        reset_state();
+        del_top_meta_list();
+
+        if (io_buffer) {
+            free(reinterpret_cast<void*>(io_buffer));
+        }
+    }
+    
     // =================================================================
     // for token-by-token parsing. If a discard or metadata is parsed,
     // attempt to get the following value
@@ -37,6 +50,11 @@ namespace edn
     {
         VALUE token = EDNT_EOF_CONST;
 
+        // buffer if reading from an IO
+        if (core_io || (read_io != Qnil)) {
+            fill_buf();
+        }
+
         while (!is_eof())
         {
             // fetch a token. If it's metadata or discard
@@ -70,6 +88,11 @@ namespace edn
         }
         // but clear any metadata on the first
         metadata.top()->clear();
+
+        // clean up
+        core_io = NULL;
+        read_io = Qnil;
+        p = pe = eof = NULL;
     }
 
     //
@@ -83,6 +106,78 @@ namespace edn
         eof = pe;
     }
 
+    void Parser::set_source(FILE* fp)
+    {
+        reset_state();
+        core_io = fp;
+    }
+
+    void Parser::set_source(VALUE str_io)
+    {
+        reset_state();
+        read_io = str_io;
+    }
+
+    //
+    // for IO sources, read and fill a buffer
+    void Parser::fill_buf()
+    {
+        std::string str_buf;
+
+        // read as much data available
+        if (core_io) {
+            // ruby core IO types
+            char c;
+            while (1)
+            {
+                c = fgetc(core_io);
+                if (c == EOF) {
+                    break;
+                }
+                str_buf += c;
+            }
+
+        } else if (read_io != Qnil) {
+            // StringIO, etc. Call read() from ruby side
+            VALUE v = ruby_io_read(read_io);
+            if (TYPE(v) == T_STRING) {
+                str_buf.assign( StringValuePtr(v), RSTRING_LEN(v));
+            }
+        }
+
+        // set the buffer to read from
+        if (str_buf.length() > 0) {
+            // first time when io_buffer is NULL, pe & p = 0
+            uintmax_t new_length = ((uintmax_t) (pe - p)) + str_buf.length();
+            char* start = NULL;
+
+            // allocate or extend storage needed
+            if (!io_buffer) {
+                io_buffer = reinterpret_cast<char*>(malloc(new_length));
+                start = io_buffer;
+            } else if (io_buffer_len < new_length) {
+                // resize the buffer
+                realloc(reinterpret_cast<void*>(io_buffer), new_length);
+            }
+
+            if (!start) {
+                // appending to the buffer but move the data not yet
+                // parsed first to the front
+                memmove(io_buffer, p, pe - p);
+                start = io_buffer + (pe - p);
+            }
+            
+            // and copy
+            memcpy(start, str_buf.c_str(), str_buf.length()); 
+            io_buffer_len = new_length;
+
+            // set ragel state
+            p = io_buffer;
+            pe = p + new_length;
+            eof = pe;
+        }
+    }
+    
 
     // =================================================================
     // work-around for idiotic rb_protect convention in order to avoid
@@ -188,6 +283,15 @@ namespace edn
     }
 
 
+    //
+    // read from a StringIO - expensive!!!
+    //
+    VALUE Parser::ruby_io_read(VALUE io)
+    {
+        prot_args args(io, EDNT_READ_METHOD);
+        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+    }
+
     //
     // copies the string data, unescaping any present values that need to be replaced
     //
@@ -198,7 +302,7 @@ namespace edn
             std::string buf;
 
             if (encode) {
-                if (!to_utf8(p_start, p_end - p_start, buf))
+                if (!util::to_utf8(p_start, p_end - p_start, buf))
                     return false;
             }
             else {

data/ext/edn_turbo/edn_parser_util.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+
+namespace edn
+{
+    namespace util
+    {
+        bool to_utf8(const char *s, std::size_t len, std::string& rslt);
+    }
+}

data/ext/edn_turbo/edn_parser_util_unicode.cc

@@ -0,0 +1,32 @@
+#include <string>
+
+//
+// needed to define this in its own file because icu and ruby have
+// differing definitions for Uchar and the compiler complains
+//
+#include <unicode/utypes.h>
+#include <unicode/ustring.h>
+#include <unicode/ucnv.h>
+
+#include "edn_parser_util.h"
+
+namespace edn
+{
+    namespace util
+    {
+        //
+        // unescapes any values that need to be replaced, saves it to utf8
+        //
+        bool to_utf8(const char *s, std::size_t len, std::string& rslt)
+        {
+            icu::UnicodeString ustr(s, len);
+
+            if (ustr.isBogus()) {
+                return false;
+            }
+
+            ustr.unescape().toUTF8String(rslt);
+            return true;
+        }
+    }
+}

data/ext/edn_turbo/main.cc

@@ -1,12 +1,13 @@
 #include <signal.h>
 #include <iostream>
 #include <clocale>
-
 #include <cstring>
 
+#include <ruby/ruby.h>
+#include <ruby/io.h>
+
 #include "edn_parser.h"
 
-#include <ruby/ruby.h>
 
 
 namespace edn {
@@ -25,6 +26,7 @@ namespace edn {
 
     VALUE EDNT_STRING_TO_I_METHOD       = Qnil;
     VALUE EDNT_STRING_TO_F_METHOD       = Qnil;
+    VALUE EDNT_READ_METHOD              = Qnil;
 
     // returned when EOF - defined as a constant in EDN module
     VALUE EDNT_EOF_CONST                = Qnil;
@@ -49,7 +51,7 @@ namespace edn {
         Data_Get_Struct( self, edn::Parser, p );
         return p;
     }
-
+    static VALUE set_source(VALUE self, VALUE data);
 
     //
     // Called by the constructor - sets the source if passed.
@@ -57,11 +59,8 @@ namespace edn {
     {
         Parser* p = get_parser(self);
 
-        if (argc > 0)
-        {
-            const char* stream = StringValueCStr(argv[0]);
-            if (stream)
-                p->set_source( stream, std::strlen(stream) );
+        if (argc > 0) {
+            set_source( self, argv[0] );
         }
         return self;
     }
@@ -72,9 +71,49 @@ namespace edn {
     {
         Parser* p = get_parser(self);
 
-        const char* stream = StringValueCStr(data);
-        if (stream)
-            p->set_source( stream, std::strlen(stream) );
+        switch (TYPE(data))
+        {
+          case T_STRING:
+          {
+              const char* stream = StringValueCStr(data);
+              if (stream) {
+                  p->set_source( stream, std::strlen(stream) );
+              }
+              break;
+          }
+          case T_FILE:
+          {
+              // extract the stream pointer
+              rb_io_t* fptr = RFILE(data)->fptr;
+              if (!fptr) {
+                  rb_raise(rb_eRuntimeError, "Ruby IO - fptr is NULL");
+              }
+
+              rb_io_check_char_readable(fptr);
+
+              FILE* fp = rb_io_stdio_file(fptr);
+              if (!fp) {
+                  rb_raise(rb_eRuntimeError, "Ruby IO - fptr->fp is NULL");
+              }
+
+              p->set_source(fp);
+              break;
+          }
+          case T_DATA:
+          {
+              // StringIO or some other IO not part of the ruby core -
+              // this is very inefficient as it'll require read()
+              // calls from the ruby side (involves a lot of data
+              // wrapping, etc)
+              if (rb_respond_to(data, EDNT_READ_METHOD)) {
+                  p->set_source(data);
+                  break;
+              }
+          }
+          default:
+              rb_raise(rb_eRuntimeError, "set_source expected String, core IO, or IO that responds to read()");
+              break;
+        }
 
         return self;
     }
@@ -90,9 +129,13 @@ namespace edn {
     // parses an entire stream
     static VALUE read(VALUE self, VALUE data)
     {
+        if (TYPE(data) != T_STRING) {
+            rb_raise(rb_eTypeError, "Expected String data");
+        }
         const char* stream = StringValueCStr(data);
-        if (stream)
+        if (stream) {
             return get_parser(self)->parse(stream, std::strlen(stream) );
+        }
         return Qnil;
     }
 
@@ -125,8 +168,7 @@ void Init_edn_turbo(void)
 
     // pass things back as utf-8
     if (!setlocale( LC_ALL, "" )) {
-        std::cerr << "Error setting locale" << std::endl;
-        return;
+        rb_raise(rb_eRuntimeError, "Extension init error calling setlocale() - It appears your system's locale is not configured correctly.\n");
     }
 
     edn::rb_mEDNT = rb_define_module("EDNT");
@@ -152,6 +194,7 @@ void Init_edn_turbo(void)
 
     edn::EDNT_STRING_TO_I_METHOD       = rb_intern("to_i");
     edn::EDNT_STRING_TO_F_METHOD       = rb_intern("to_f");
+    edn::EDNT_READ_METHOD              = rb_intern("read");
 
     // so we can return EOF directly
     VALUE edn_module                   = rb_const_get(rb_cObject, edn::EDN_MODULE_SYMBOL);

data/lib/edn_turbo/version.rb

@@ -1,4 +1,4 @@
 module EDNT
-  VERSION = '0.4.1'
-  RELEASE_DATE = %q{2016-07-21}
+  VERSION = '0.5.0'
+  RELEASE_DATE = '2016-11-18'
 end