edn_turbo 0.3.0 → 0.3.1

data/ext/edn_turbo/edn_parser.h

@@ -1,9 +1,9 @@
-#ifndef EDN_TURBO_PARSER_H
-#define EDN_TURBO_PARSER_H
+#ifndef EDN_RAGEL_PARSER_H
+#define EDN_RAGEL_PARSER_H
 
 #include <string>
 #include <sstream>
-#include <stack>
+#include <vector>
 
 #include <ruby/ruby.h>
 
@@ -14,8 +14,10 @@ namespace edn
     extern VALUE EDNT_MAKE_EDN_SYMBOL;
     extern VALUE EDNT_MAKE_SET_METHOD;
     extern VALUE EDNT_TAGGED_ELEM;
+    extern VALUE EDNT_BIND_META_TO_VALUE;
     extern VALUE EDNT_STR_INT_TO_BIGNUM;
     extern VALUE EDNT_STR_DBL_TO_BIGNUM;
+    extern VALUE EDNT_EOF;
 
     //
     // C-extension EDN Parser class representation
@@ -24,11 +26,15 @@ namespace edn
     public:
         Parser() : p(NULL), pe(NULL), eof(NULL), line_number(1) { }
 
+        // change input source
         void set_source(const char* src, std::size_t len);
 
-        bool is_eof() const { return (p != NULL && p == pe); }
+        bool is_eof() const { return (p == eof); }
+
+        // parses an entire stream
         VALUE parse(const char* s, std::size_t len);
-        //        VALUE read(const std::string& data) { return parse(data.c_str(), data.length()); }
+
+        // returns the next element in the current stream
         VALUE next();
 
         static void throw_error(int error);
@@ -39,9 +45,10 @@ namespace edn
         const char* pe;
         const char* eof;
         std::size_t line_number;
-        std::stack<VALUE> discard;
+        std::vector<VALUE> discard;
+        std::vector<VALUE> metadata;
 
-        void reset();
+        void reset_state();
 
         const char* parse_value   (const char *p, const char *pe, VALUE& v);
         const char* parse_string  (const char *p, const char *pe, VALUE& v);
@@ -58,6 +65,9 @@ namespace edn
         const char* parse_set     (const char *p, const char *pe, VALUE& v);
         const char* parse_discard (const char *p, const char *pe);
         const char* parse_tagged  (const char *p, const char *pe, VALUE& v);
+        const char* parse_meta    (const char *p, const char *pe);
+
+        bool parse_next(VALUE& value);
 
         // defined in edn_parser_unicode.cc
         static bool to_utf8(const char *s, std::size_t len, std::string& rslt);
@@ -73,6 +83,10 @@ namespace edn
         static VALUE make_ruby_set(VALUE elems);
         static VALUE tagged_element(VALUE name, VALUE data);
 
+        // metadata
+        VALUE ruby_meta();
+        VALUE bind_meta_to_value(VALUE value);
+
         // utility method to convert a primitive in string form to a
         // ruby type
         template <class T>

data/ext/edn_turbo/edn_parser.rl

@@ -1,6 +1,6 @@
 #include <iostream>
 #include <string>
-#include <stack>
+#include <vector>
 #include <exception>
 
 #include "edn_parser.h"
@@ -25,18 +25,19 @@
 
         operators      = [/\.\*!_\?$%&<>\=+\-];
         symbol_start   = alpha;
-        symbol_chars   = symbol_start | digit | [\#:_\-\.];
+        symbol_chars   = symbol_start | digit | [\#:_\-\.\'];
 
         begin_dispatch = '#';
         begin_keyword  = ':';
         begin_char     = '\\';
-        begin_value    = alnum | [:\"\{\[\(\\\#] | operators;
-        begin_symbol   = symbol_start;
         begin_vector   = '[';
         begin_map      = '{';
         begin_list     = '(';
+        begin_meta     = '^';
         string_delim   = '"';
         begin_number   = digit;
+        begin_value    = alnum | [:\"\{\[\(\\\#^] | operators;
+        begin_symbol   = symbol_start;
 
         symbol_name    = symbol_start (symbol_chars)*;
         symbol         = (symbol_name ('/' symbol_name)?);
@@ -145,6 +146,12 @@
         if (np == NULL) { fhold; fbreak; } else fexec np;
     }
 
+    action parse_meta {
+        // ^
+        const char *np = parse_meta(fpc, pe);
+        if (np == NULL) { fhold; fbreak; } else fexec np;
+    }
+
     action parse_dispatch {
         // handles tokens w/ leading # ("#_", "#{", and tagged elems)
         const char *np = parse_dispatch(fpc + 1, pe, v);
@@ -162,6 +169,7 @@
              begin_vector >parse_vector |
              begin_list >parse_list |
              begin_map >parse_map |
+             begin_meta >parse_meta |
              begin_dispatch >parse_dispatch
             ) %*exit;
 }%%
@@ -223,7 +231,6 @@ const char *edn::Parser::parse_value(const char *p, const char *pe, VALUE& v)
 
 const char* edn::Parser::parse_string(const char *p, const char *pe, VALUE& v)
 {
-    //    std::cerr << __FUNCTION__ << "   -  p: '" << p << "'" << std::endl;
     static const char* EDN_TYPE = "string";
     int cs;
     bool encode = false;
@@ -551,7 +558,7 @@ const char* edn::Parser::parse_symbol(const char *p, const char *pe, VALUE& s)
             // object is not meant to be kept due to a #_ so don't
             // push it into the list of elements
             if (!discard.empty()) {
-                discard.pop();
+                discard.pop_back();
             }
             else {
                 // otherwise we add it to the list of elements for the
@@ -834,17 +841,23 @@ const char* edn::Parser::parse_set(const char *p, const char *pe, VALUE& v)
             // this token is to be discard it so store it in the
             // discard stack - we really don't need to save it so this
             // could be simplified
-            discard.push(v);
+            discard.push_back(v);
             fexec np;
         } else {
             fhold; fbreak;
         }
     }
 
+    action discard_err {
+        std::stringstream s;
+        s << "discard sequence without element to discard";
+        error(__FUNCTION__, s.str());
+        fhold; fbreak;
+    }
 
     main := begin_discard ignore* (
                                    begin_value >discard_value
-                                   ) @exit;
+                                   ) @err(discard_err) @exit;
 }%%
 
 
@@ -942,6 +955,51 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, VALUE& v)
 
 
 
+// ============================================================
+// metadata - looks like ruby just discards this but we'll track it
+// and provide a means to retrive after each parse op - might be
+// useful?
+//
+%%{
+    machine EDN_meta;
+    include EDN_common;
+
+    write data;
+
+    action parse_meta {
+        const char *np = parse_value(fpc, pe, v);
+        if (np) { fexec np; } else { fhold; fbreak; }
+    }
+
+    main := begin_meta (
+                        begin_value >parse_meta
+                        ) @exit;
+}%%
+
+
+const char* edn::Parser::parse_meta(const char *p, const char *pe)
+{
+    int cs;
+    VALUE v;
+
+    %% write init;
+    %% write exec;
+
+    if (cs >= EDN_meta_first_final) {
+        metadata.push_back(v);
+        return p + 1;
+    }
+    else if (cs == EDN_meta_error) {
+        error(__FUNCTION__, *p);
+        return pe;
+    }
+    else if (cs == EDN_meta_en_main) {} // silence ragel warning
+
+    return NULL;
+}
+
+
+
 // ============================================================
 // parses entire input but expects single valid token at the
 // top-level, therefore, does not tokenize source stream
@@ -953,8 +1011,21 @@ const char* edn::Parser::parse_tagged(const char *p, const char *pe, VALUE& v)
     write data;
 
     action parse_value {
+        // save the count of metadata items before we parse this value
+        // so we can determine if we've read another metadata value or
+        // an actual data item
+        std::size_t meta_size = metadata.size();
         const char* np = parse_value(fpc, pe, result);
-        if (np == NULL) { fexec pe; fbreak; } else fexec np;
+        if (np == NULL) { fexec pe; fbreak; } else {
+            // if we have metadata saved and it matches the count we
+            // saved before we parsed a value, then we must bind the
+            // metadata sequence to it
+            if (!metadata.empty() && metadata.size() == meta_size) {
+                // this will empty the metadata sequence too
+                result = bind_meta_to_value(result);
+            }
+            fexec np;
+        }
     }
 
     element       = begin_value >parse_value;
@@ -970,23 +1041,17 @@ VALUE edn::Parser::parse(const char* src, std::size_t len)
     int cs;
     VALUE result = Qnil;
 
-    // reset line counter & discard stack
-    reset();
-
     %% write init;
-    p = src;
-    pe = p + len;
-    eof = pe;
+    set_source(src, len);
     %% write exec;
 
     if (cs == EDN_parser_error) {
-        error(__FUNCTION__, *p);
-        return Qnil;
+        if (p)
+            error(__FUNCTION__, *p);
+        return EDNT_EOF;
     }
     else if (cs == EDN_parser_first_final) {
-        // whole source is parsed so reset
         p = pe = eof = NULL;
-        reset();
     }
     else if (cs == EDN_parser_en_main) {} // silence ragel warning
     return result;
@@ -1000,32 +1065,64 @@ VALUE edn::Parser::parse(const char* src, std::size_t len)
     machine EDN_tokens;
     include EDN_common;
 
-    write data noerror nofinal;
+    write data nofinal;
 
     action parse_value {
-        const char* np = parse_value(fpc, pe, result);
-        if (np == NULL) { fhold; fbreak; } else { fexec np; }
-    }
+        // we won't know if we've parsed a discard or a metadata until
+        // after parse_value() is done. Save the current number of
+        // elements in the metadata sequence; then we can check if it
+        // grew or if the discard sequence grew
+        meta_size = metadata.size();
 
-    element       = begin_value >parse_value;
+        const char* np = parse_value(fpc, pe, value);
+
+        if (np == NULL) { fhold; fbreak; } else {
+            if (metadata.size() > 0) {
+                // was anotheran additional metadata entry read? if
+                // so, don't return a value
+                if (metadata.size() > meta_size) {
+                    is_value = false;
+                }
+                else {
+                    // a value was read and there's a pending metadata
+                    // sequence. Bind them.
+                    value = bind_meta_to_value(value);
+                }
+            } else if (!discard.empty()) {
+                // a discard read. Don't return a value
+                is_value = false;
+            }
+            fexec np;
+        }
+    }
 
-    main := ignore* element ignore*;
+    main := ignore* begin_value >parse_value ignore*;
 }%%
 
 
 //
 //
-VALUE edn::Parser::next()
+bool edn::Parser::parse_next(VALUE& value)
 {
-    VALUE result = Qnil;
     int cs;
+    bool is_value = true;
+    // need to track metadada read and bind it to the next value read
+    // - but must account for sequences of metadata values
+    std::size_t meta_size;
+
+    // clear any previously saved discards; only track if read during
+    // this op
+    discard.clear();
 
     %% write init;
     %% write exec;
 
-    if (cs == EDN_tokens_en_main) {} // silence ragel warning
+    if (cs == EDN_parser_error) {
+        value = EDNT_EOF;
+    }
+    else if (cs == EDN_tokens_en_main) {} // silence ragel warning
 
-    return result;
+    return is_value;
 }
 
 

data/ext/edn_turbo/edn_parser_util.cc

@@ -1,5 +1,4 @@
 #include <iostream>
-#include <iomanip>
 #include <string>
 #include <limits>
 #include <exception>
@@ -26,20 +25,42 @@ namespace edn
 
 
     // =================================================================
+    // for token-by-token parsing. If a discard or metadata is parsed,
+    // attempt to get the following value
+    //
+    VALUE Parser::next()
+    {
+        VALUE token = EDNT_EOF;
+
+        while (!is_eof())
+        {
+            // fetch a token. If it's metadata or discard
+            VALUE v;
+            if (parse_next(v))
+            {
+                // valid token
+                token = v;
+                break;
+            }
+        }
+
+        return token;
+    }
+
     // reset parsing state
     //
-    void Parser::reset()
+    void Parser::reset_state()
     {
         line_number = 1;
-        while (!discard.empty())
-            discard.pop();
+        discard.clear();
+        metadata.clear();
     }
 
     //
     // set a new source
     void Parser::set_source(const char* src, std::size_t len)
     {
-        reset();
+        reset_state();
         // set ragel state
         p = src;
         pe = src + len;
@@ -55,16 +76,20 @@ namespace edn
 
     // we're using at most 2 args
     struct prot_args {
-        prot_args(ID m, VALUE arg) :
-            method(m), count(1) {
+        prot_args(ID r, ID m, VALUE arg) :
+            receiver(r), method(m), count(1) {
             args[0] = arg;
         }
-        prot_args(ID m, VALUE arg1, VALUE arg2) :
-            method(m), count(2) {
+        prot_args(ID r, ID m, VALUE arg1, VALUE arg2) :
+            receiver(r), method(m), count(2) {
             args[0] = arg1;
             args[1] = arg2;
         }
 
+        VALUE call() const { return rb_funcall2( receiver, method, count, args ); }
+
+    private:
+        ID receiver;
         ID method;
         VALUE count;
         VALUE args[2];
@@ -73,8 +98,10 @@ namespace edn
     // this allows us to wrap with rb_protect()
     static inline VALUE edn_wrap_funcall2( VALUE arg )
     {
-        prot_args* a = reinterpret_cast<prot_args*>(arg);
-        return rb_funcall2( edn::rb_mEDNT, a->method, a->count, a->args );
+        const prot_args* a = reinterpret_cast<const prot_args*>(arg);
+        if (a)
+            return a->call();
+        return Qnil;
     }
 
     static inline VALUE edn_prot_rb_funcall( edn_rb_f_type func, VALUE args )
@@ -113,7 +140,7 @@ namespace edn
 
         // value is outside of range of long type. Use ruby to convert it
         VALUE rb_s = edn_prot_rb_new_str( str );
-        prot_args args(EDNT_STR_INT_TO_BIGNUM, rb_s);
+        prot_args args(rb_mEDNT, EDNT_STR_INT_TO_BIGNUM, rb_s);
         return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
@@ -127,7 +154,7 @@ namespace edn
         }
 
         // value is outside of range of long type. Use ruby to convert it
-        prot_args args(EDNT_STR_DBL_TO_BIGNUM, edn_prot_rb_new_str(str));
+        prot_args args(rb_mEDNT, EDNT_STR_DBL_TO_BIGNUM, edn_prot_rb_new_str(str));
         return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
@@ -193,7 +220,7 @@ namespace edn
     // get a set representation from the ruby side. See edn_turbo.rb
     VALUE Parser::make_edn_symbol(VALUE sym)
     {
-        prot_args args(edn::EDNT_MAKE_EDN_SYMBOL, sym);
+        prot_args args(rb_mEDNT, EDNT_MAKE_EDN_SYMBOL, sym);
         return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
@@ -201,7 +228,7 @@ namespace edn
     // get a set representation from the ruby side. See edn_turbo.rb
     VALUE Parser::make_ruby_set(VALUE elems)
     {
-        prot_args args(edn::EDNT_MAKE_SET_METHOD, elems);
+        prot_args args(rb_mEDNT, EDNT_MAKE_SET_METHOD, elems);
         return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
@@ -209,11 +236,38 @@ namespace edn
     // get an object representation from the ruby side using the given symbol name
     VALUE Parser::tagged_element(VALUE name, VALUE data)
     {
-        prot_args args(edn::EDNT_TAGGED_ELEM, name, data);
+        prot_args args(rb_mEDNT, EDNT_TAGGED_ELEM, name, data);
         return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
 
+    // =================================================================
+    // METADATA
+    //
+    // returns an array of metadata value(s) saved in reverse order
+    // (right to left) - the ruby side will interpret this
+    VALUE Parser::ruby_meta()
+    {
+        VALUE m = rb_ary_new();
+
+        while (!metadata.empty()) {
+            rb_ary_push(m, metadata.back());
+            metadata.pop_back();
+        }
+
+        return m;
+    }
+
+    //
+    // calls the ruby-side bind_meta to bind the metadata to the value
+    VALUE Parser::bind_meta_to_value(VALUE value)
+    {
+        prot_args args(rb_mEDNT, EDNT_BIND_META_TO_VALUE, value, ruby_meta());
+        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+    }
+
+
+    // =================================================================
     //
     // error reporting
     void Parser::throw_error(int error)

data/ext/edn_turbo/main.cc

@@ -12,11 +12,15 @@ namespace edn {
     VALUE rb_mEDNT;
 
     // methods on the ruby side we'll call from here
-    VALUE EDNT_MAKE_EDN_SYMBOL = Qnil;
-    VALUE EDNT_MAKE_SET_METHOD = Qnil;
-    VALUE EDNT_TAGGED_ELEM = Qnil;
-    VALUE EDNT_STR_INT_TO_BIGNUM = Qnil;
-    VALUE EDNT_STR_DBL_TO_BIGNUM = Qnil;
+    VALUE EDNT_MAKE_EDN_SYMBOL    = Qnil;
+    VALUE EDNT_MAKE_SET_METHOD    = Qnil;
+    VALUE EDNT_TAGGED_ELEM        = Qnil;
+    VALUE EDNT_BIND_META_TO_VALUE = Qnil;
+    VALUE EDNT_STR_INT_TO_BIGNUM  = Qnil;
+    VALUE EDNT_STR_DBL_TO_BIGNUM  = Qnil;
+
+    // returned when EOF - defined as a constant in EDN module
+    VALUE EDNT_EOF                = Qnil;
 
     //
     // wrappers to hook the class w/ the C-api
@@ -82,7 +86,9 @@ namespace edn {
     static VALUE read(VALUE self, VALUE data)
     {
         const char* stream = StringValueCStr(data);
-        return get_parser(self)->parse(stream, std::strlen(stream) );
+        if (stream)
+            return get_parser(self)->parse(stream, std::strlen(stream) );
+        return Qnil;
     }
 
     //
@@ -130,11 +136,15 @@ void Init_edn_turbo(void)
     rb_define_method(rb_cParser, "ext_next", (VALUE(*)(ANYARGS)) &edn::next, 0 );
 
     // bind ruby methods we'll call - these should be defined in edn_turbo.rb
-    edn::EDNT_MAKE_EDN_SYMBOL = rb_intern("make_edn_symbol");
-    edn::EDNT_MAKE_SET_METHOD = rb_intern("make_set");
-    edn::EDNT_TAGGED_ELEM = rb_intern("tagged_element");
-    edn::EDNT_STR_INT_TO_BIGNUM = rb_intern("string_int_to_bignum");
-    edn::EDNT_STR_DBL_TO_BIGNUM = rb_intern("string_double_to_bignum");
+    edn::EDNT_MAKE_EDN_SYMBOL    = rb_intern("make_edn_symbol");
+    edn::EDNT_MAKE_SET_METHOD    = rb_intern("make_set");
+    edn::EDNT_TAGGED_ELEM        = rb_intern("tagged_element");
+    edn::EDNT_BIND_META_TO_VALUE = rb_intern("bind_metadata_to_value");
+    edn::EDNT_STR_INT_TO_BIGNUM  = rb_intern("string_int_to_bignum");
+    edn::EDNT_STR_DBL_TO_BIGNUM  = rb_intern("string_double_to_bignum");
+
+    // so we can return EOF directly
+    edn::EDNT_EOF = rb_const_get(edn::rb_mEDNT, rb_intern("EOF"));
 
     // import whatever else we've defined in the ruby side
     rb_require("edn_turbo/edn_parser");

data/lib/edn_turbo/edn_parser.rb

@@ -1,3 +1,5 @@
+require 'edn'
+
 module EDNT
 
   EOF = Object.new
@@ -13,7 +15,6 @@ module EDNT
 
     # token-by-token read
     def read
-      return EOF if ext_eof
       ext_next
     end
 

data/lib/edn_turbo/version.rb

@@ -1,4 +1,4 @@
 module EDNT
-  VERSION = '0.3.0'
-  RELEASE_DATE = %q{2015-06-12}
+  VERSION = '0.3.1'
+  RELEASE_DATE = %q{2015-06-18}
 end

data/lib/edn_turbo.rb

@@ -91,4 +91,34 @@ module EDNT
     str.to_f
   end
 
+  # ============================================================================
+  # emulate EDN::Metadata
+  module Metadata
+    attr_accessor :metadata
+  end
+
+  # ----------------------------------------------------------------------------
+  # bind the given meta to the value
+  #
+  def self.bind_metadata_to_value(value, ext_meta)
+
+    meta = ext_meta
+
+    metadata = meta.reduce({}) do |acc, m|
+      case m
+      when Symbol then acc.merge(m => true)
+      when EDN::Type::Symbol then acc.merge(:tag => m)
+      else acc.merge(m)
+      end
+    end
+
+    if !metadata.empty?
+      value.extend Metadata
+      value.metadata = metadata
+    end
+
+    value
+  end
+
+
 end # EDN namespace

data/test/test_output_diff.rb

@@ -84,6 +84,18 @@ class EDNT_Test < Minitest::Test
 
   end
 
+  def test_symbol
+
+    check_file('test/symbol.edn',
+               [
+                 EDN::Type::Symbol.new("asymbol"),
+                 EDN::Type::Symbol.new("with'_a_'"),
+                 EDN::Type::Symbol.new("with.123"),
+               ]
+               )
+
+  end
+
   def test_unicode
 
     check_file('test/unicode.edn',
@@ -129,6 +141,17 @@ class EDNT_Test < Minitest::Test
               )
   end
 
+
+  def test_metadata
+    f = EDNT::read(File.open('test/metadata.edn').read)
+    assert_equal([98.6, 99.7], f)
+    assert_equal({:doc=>"This is my vector", :rel=>:temps}, f.metadata)
+
+    f = EDNT::read(File.open('test/metadata2.edn').read)
+    assert_equal([1, 2], f)
+    assert_equal({:foo=>true, :tag=>EDN::Type::Symbol.new('String'), :bar=>2}, f.metadata)
+  end
+
   #
   # for testing tagged element #edn_turbo/test_tagged
   class Tagged
@@ -151,7 +174,7 @@ class EDNT_Test < Minitest::Test
     assert_equal([345, :a], @parser.parse('#edn_turbo/test_tagged { :item 345 :other :a }'))
   end
 
-  def test_symbols
+  def test_operators
 
     check_file('test/operators.edn',
                [EDN::Type::Symbol.new('/'),
@@ -168,6 +191,10 @@ class EDNT_Test < Minitest::Test
                 EDN::Type::Symbol.new('='),
                 EDN::Type::Symbol.new('-'),
                 EDN::Type::Symbol.new('+'),
+#                [1, EDN::Type::Symbol.new('/'), 2],
+#                [3, EDN::Type::Symbol.new('/'), 4],
+                [5, EDN::Type::Symbol.new('/'), 6],
+                [7, EDN::Type::Symbol.new('/'), 8]
                ]
               )