edn_turbo 0.3.1 → 0.3.2

data/ext/edn_turbo/edn_parser_util.cc

@@ -1,5 +1,7 @@
 #include <iostream>
 #include <string>
+#include <stack>
+#include <vector>
 #include <limits>
 #include <exception>
 
@@ -30,18 +32,23 @@ namespace edn
     //
     VALUE Parser::next()
     {
-        VALUE token = EDNT_EOF;
+        VALUE token = EDNT_EOF_CONST;
 
         while (!is_eof())
         {
             // fetch a token. If it's metadata or discard
-            VALUE v;
-            if (parse_next(v))
-            {
+            VALUE v = EDNT_EOF_CONST;
+            eTokenState state = parse_next(v);
+
+            if (state == TOKEN_OK) {
                 // valid token
                 token = v;
                 break;
             }
+            else if (state == TOKEN_ERROR) {
+                token = EDNT_EOF_CONST;
+                break;
+            }
         }
 
         return token;
@@ -53,7 +60,13 @@ namespace edn
     {
         line_number = 1;
         discard.clear();
-        metadata.clear();
+
+        // remove any remaining levels except for the first
+        while (metadata.size() > 1) {
+            del_top_meta_list();
+        }
+        // but clear any metadata on the first
+        metadata.top()->clear();
     }
 
     //
@@ -76,20 +89,27 @@ namespace edn
 
     // we're using at most 2 args
     struct prot_args {
-        prot_args(ID r, ID m, VALUE arg) :
+        prot_args(VALUE r, ID m) :
+            receiver(r), method(m), count(0) {
+        }
+        prot_args(VALUE r, ID m, VALUE arg) :
             receiver(r), method(m), count(1) {
             args[0] = arg;
         }
-        prot_args(ID r, ID m, VALUE arg1, VALUE arg2) :
+        prot_args(VALUE r, ID m, VALUE arg1, VALUE arg2) :
             receiver(r), method(m), count(2) {
             args[0] = arg1;
             args[1] = arg2;
         }
 
-        VALUE call() const { return rb_funcall2( receiver, method, count, args ); }
+        VALUE call() const {
+            return ((count == 0) ?
+                    rb_funcall( receiver, method, 0 ) :
+                    rb_funcall2( receiver, method, count, args ));
+        }
 
     private:
-        ID receiver;
+        VALUE receiver;
         ID method;
         VALUE count;
         VALUE args[2];
@@ -133,29 +153,35 @@ namespace edn
     // hold, call into ruby to get a big num
     VALUE Parser::integer_to_ruby(const char* str, std::size_t len)
     {
-        if (len < LL_max_chars)
+        if (str[len-1] == 'M' || len >= LL_max_chars)
         {
-            return LONG2NUM(buftotype<long>(str, len));
+            std::string buf(str, len);
+            VALUE vs = edn_prot_rb_new_str(buf.c_str());
+            prot_args args(vs, EDNT_STRING_TO_I_METHOD);
+            return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
         }
 
-        // value is outside of range of long type. Use ruby to convert it
-        VALUE rb_s = edn_prot_rb_new_str( str );
-        prot_args args(rb_mEDNT, EDNT_STR_INT_TO_BIGNUM, rb_s);
-        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+        return LONG2NUM(buftotype<long>(str, len));
     }
 
     //
     // as above.. TODO: check exponential..
     VALUE Parser::float_to_ruby(const char* str, std::size_t len)
     {
-        if (len < LD_max_chars)
+        if (str[len-1] == 'M' || len >= LD_max_chars)
         {
-            return rb_float_new(buftotype<double>(str, len));
+            std::string buf(str, len);
+            VALUE vs = edn_prot_rb_new_str(buf.c_str());
+
+            if (str[len-1] == 'M') {
+                return Parser::make_edn_type(EDNT_MAKE_BIG_DECIMAL_METHOD, vs);
+            }
+
+            prot_args args(vs, EDNT_STRING_TO_F_METHOD);
+            return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
         }
 
-        // value is outside of range of long type. Use ruby to convert it
-        prot_args args(rb_mEDNT, EDNT_STR_DBL_TO_BIGNUM, edn_prot_rb_new_str(str));
-        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+        return rb_float_new(buftotype<double>(str, len));
     }
 
 
@@ -218,25 +244,15 @@ namespace edn
 
     //
     // get a set representation from the ruby side. See edn_turbo.rb
-    VALUE Parser::make_edn_symbol(VALUE sym)
+    VALUE Parser::make_edn_type(ID method, VALUE sym)
     {
-        prot_args args(rb_mEDNT, EDNT_MAKE_EDN_SYMBOL, sym);
+        prot_args args(rb_mEDNT, method, sym);
         return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
-    //
-    // get a set representation from the ruby side. See edn_turbo.rb
-    VALUE Parser::make_ruby_set(VALUE elems)
+    VALUE Parser::make_edn_type(ID method, VALUE name, VALUE data)
     {
-        prot_args args(rb_mEDNT, EDNT_MAKE_SET_METHOD, elems);
-        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
-    }
-
-    //
-    // get an object representation from the ruby side using the given symbol name
-    VALUE Parser::tagged_element(VALUE name, VALUE data)
-    {
-        prot_args args(rb_mEDNT, EDNT_TAGGED_ELEM, name, data);
+        prot_args args(rb_mEDNT, method, name, data);
         return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
     }
 
@@ -248,22 +264,15 @@ namespace edn
     // (right to left) - the ruby side will interpret this
     VALUE Parser::ruby_meta()
     {
-        VALUE m = rb_ary_new();
+        VALUE m_ary = rb_ary_new();
 
-        while (!metadata.empty()) {
-            rb_ary_push(m, metadata.back());
-            metadata.pop_back();
+        // pop from the back of the top-most list
+        while (!metadata.top()->empty()) {
+            rb_ary_push(m_ary, metadata.top()->back());
+            metadata.top()->pop_back();
         }
 
-        return m;
-    }
-
-    //
-    // calls the ruby-side bind_meta to bind the metadata to the value
-    VALUE Parser::bind_meta_to_value(VALUE value)
-    {
-        prot_args args(rb_mEDNT, EDNT_BIND_META_TO_VALUE, value, ruby_meta());
-        return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+        return m_ary;
     }
 
 

data/ext/edn_turbo/main.cc

@@ -12,15 +12,18 @@ namespace edn {
     VALUE rb_mEDNT;
 
     // methods on the ruby side we'll call from here
-    VALUE EDNT_MAKE_EDN_SYMBOL    = Qnil;
-    VALUE EDNT_MAKE_SET_METHOD    = Qnil;
-    VALUE EDNT_TAGGED_ELEM        = Qnil;
-    VALUE EDNT_BIND_META_TO_VALUE = Qnil;
-    VALUE EDNT_STR_INT_TO_BIGNUM  = Qnil;
-    VALUE EDNT_STR_DBL_TO_BIGNUM  = Qnil;
+    VALUE EDNT_MAKE_SYMBOL_METHOD       = Qnil;
+    VALUE EDNT_MAKE_LIST_METHOD         = Qnil;
+    VALUE EDNT_MAKE_SET_METHOD          = Qnil;
+    VALUE EDNT_MAKE_BIG_DECIMAL_METHOD  = Qnil;
+    VALUE EDNT_TAGGED_ELEM_METHOD       = Qnil;
+    VALUE EDNT_EXTENDED_VALUE_METHOD    = Qnil;
+
+    VALUE EDNT_STRING_TO_I_METHOD       = Qnil;
+    VALUE EDNT_STRING_TO_F_METHOD       = Qnil;
 
     // returned when EOF - defined as a constant in EDN module
-    VALUE EDNT_EOF                = Qnil;
+    VALUE EDNT_EOF_CONST                = Qnil;
 
     //
     // wrappers to hook the class w/ the C-api
@@ -136,16 +139,16 @@ void Init_edn_turbo(void)
     rb_define_method(rb_cParser, "ext_next", (VALUE(*)(ANYARGS)) &edn::next, 0 );
 
     // bind ruby methods we'll call - these should be defined in edn_turbo.rb
-    edn::EDNT_MAKE_EDN_SYMBOL    = rb_intern("make_edn_symbol");
-    edn::EDNT_MAKE_SET_METHOD    = rb_intern("make_set");
-    edn::EDNT_TAGGED_ELEM        = rb_intern("tagged_element");
-    edn::EDNT_BIND_META_TO_VALUE = rb_intern("bind_metadata_to_value");
-    edn::EDNT_STR_INT_TO_BIGNUM  = rb_intern("string_int_to_bignum");
-    edn::EDNT_STR_DBL_TO_BIGNUM  = rb_intern("string_double_to_bignum");
+    edn::EDNT_MAKE_SYMBOL_METHOD       = rb_intern("symbol");
+    edn::EDNT_MAKE_LIST_METHOD         = rb_intern("list");
+    edn::EDNT_MAKE_SET_METHOD          = rb_intern("set");
+    edn::EDNT_MAKE_BIG_DECIMAL_METHOD  = rb_intern("big_decimal");
+    edn::EDNT_TAGGED_ELEM_METHOD       = rb_intern("tagged_element");
+    edn::EDNT_EXTENDED_VALUE_METHOD    = rb_intern("extend_for_meta");
 
-    // so we can return EOF directly
-    edn::EDNT_EOF = rb_const_get(edn::rb_mEDNT, rb_intern("EOF"));
+    edn::EDNT_STRING_TO_I_METHOD       = rb_intern("to_i");
+    edn::EDNT_STRING_TO_F_METHOD       = rb_intern("to_f");
 
-    // import whatever else we've defined in the ruby side
-    rb_require("edn_turbo/edn_parser");
+    // so we can return EOF directly
+    edn::EDNT_EOF_CONST                = rb_const_get(edn::rb_mEDNT, rb_intern("EOF"));
 }

data/lib/edn_turbo/constants.rb

@@ -0,0 +1,14 @@
+require 'edn'
+
+module EDNT
+
+  # Object returned when there is nothing to return
+
+  NOTHING = EDN::NOTHING
+
+  # Object to return when we hit end of file. Cant be nil or :eof
+  # because either of those could be something in the EDN data.
+
+  EOF = EDN::EOF
+
+end

data/lib/edn_turbo/edn_parser.rb

@@ -1,9 +1,8 @@
 require 'edn'
+require 'bigdecimal'
 
 module EDNT
 
-  EOF = Object.new
-
   class Parser
 
     # initialize() is defined in the c-side (main.cc)
@@ -25,4 +24,8 @@ module EDNT
 
   end
 
+  def self.big_decimal(str)
+    BigDecimal.new(str)
+  end
+
 end

data/lib/edn_turbo/tags.rb

@@ -0,0 +1,46 @@
+require 'edn'
+
+module EDNT
+
+  # ----------------------------------------------------------------------------
+  # register a tagged element
+  #
+  TAGS = {
+    # built-in tagged elements
+    "inst" => lambda { |*a| DateTime.parse(*a) },
+    "uuid" => lambda { |*a| String.new(*a) }
+  }
+
+  def self.register(tag, func = nil, &block)
+    # don't allow re-registration of built-in tags
+    if tag != "inst" && tag != "uuid"
+      if block_given?
+        func = block
+      end
+
+      if func.nil?
+        func = lambda { |x| x }
+      end
+
+      if func.is_a?(Class)
+        TAGS[tag] = lambda { |*args| func.new(*args) }
+      else
+        TAGS[tag] = func
+      end
+    end
+  end
+
+  def self.unregister(tag)
+    TAGS[tag] = nil
+  end
+
+  def self.tagged_element(tag, element)
+    func = TAGS[tag]
+    if func
+      func.call(element)
+    else
+      EDN::Type::Unknown.new(tag, element)
+    end
+  end
+
+end

data/lib/edn_turbo/utils.rb

@@ -0,0 +1,34 @@
+require 'edn'
+
+module EDNT
+
+  # ============================================================================
+  # emulate EDN::Metadata
+  module Metadata
+    attr_accessor :metadata
+  end
+
+  # ----------------------------------------------------------------------------
+  # bind the given meta to the value
+  #
+  def self.extend_for_meta(value, ext_meta)
+
+    meta = ext_meta
+
+    metadata = meta.reduce({}) do |acc, m|
+      case m
+      when Symbol then acc.merge(m => true)
+      when EDN::Type::Symbol then acc.merge(:tag => m)
+      else acc.merge(m)
+      end
+    end
+
+    if !metadata.empty?
+      value.extend Metadata
+      value.metadata = metadata
+    end
+
+    value
+  end
+
+end

data/lib/edn_turbo/version.rb

@@ -1,4 +1,4 @@
 module EDNT
-  VERSION = '0.3.1'
-  RELEASE_DATE = %q{2015-06-18}
+  VERSION = '0.3.2'
+  RELEASE_DATE = %q{2015-06-29}
 end

data/lib/edn_turbo.rb

@@ -1,8 +1,10 @@
-require_relative 'edn_turbo/edn_parser'
+require_relative 'edn_turbo/constants'
+require_relative 'edn_turbo/tags'
+require_relative 'edn_turbo/utils'
 require_relative 'edn_turbo/version'
+require_relative 'edn_turbo/edn_parser'
 require_relative 'edn_turbo/edn_turbo'
 
-require 'date'
 require 'set'
 require 'edn'
 
@@ -23,102 +25,18 @@ module EDNT
   end
 
   # ----------------------------------------------------------------------------
-  # register a tagged element
-  #
-  TAGS = {
-    # built-in tagged elements
-    "inst" => lambda { |*a| DateTime.parse(*a) },
-    "uuid" => lambda { |*a| String.new(*a) }
-  }
-
-  def self.register(tag, func = nil, &block)
-    # don't allow re-registration of built-in tags
-    if tag != "inst" && tag != "uuid"
-      if block_given?
-        func = block
-      end
-
-      if func.nil?
-        func = lambda { |x| x }
-      end
-
-      if func.is_a?(Class)
-        TAGS[tag] = lambda { |*args| func.new(*args) }
-      else
-        TAGS[tag] = func
-      end
-    end
-  end
-
-  def self.unregister(tag)
-    TAGS[tag] = nil
-  end
-
-  def self.tagged_element(tag, element)
-    func = TAGS[tag]
-    if func
-      func.call(element)
-    else
-      EDN::Type::Unknown.new(tag, element)
-    end
-  end
-
-  # ----------------------------------------------------------------------------
-  # handles creation of a set from an array
-  #
-  def self.make_set(elems)
-    Set.new(elems)
-  end
-
-  # ----------------------------------------------------------------------------
-  # handles creation of an EDN::Type::Symbol
+  # handles creation of various EDN representations
   #
-  def self.make_edn_symbol(elem)
+  def self.symbol(elem)
     EDN::Type::Symbol.new(elem)
   end
 
-  # ----------------------------------------------------------------------------
-  # to create Big Ints (for now)
-  #
-  def self.string_int_to_bignum(str)
-    str.to_i
-  end
-
-  # ----------------------------------------------------------------------------
-  # to create Big Ints (for now)
-  #
-  def self.string_double_to_bignum(str)
-    str.to_f
-  end
-
-  # ============================================================================
-  # emulate EDN::Metadata
-  module Metadata
-    attr_accessor :metadata
+  def self.list(*elems)
+    EDN::Type::List.new(*elems)
   end
 
-  # ----------------------------------------------------------------------------
-  # bind the given meta to the value
-  #
-  def self.bind_metadata_to_value(value, ext_meta)
-
-    meta = ext_meta
-
-    metadata = meta.reduce({}) do |acc, m|
-      case m
-      when Symbol then acc.merge(m => true)
-      when EDN::Type::Symbol then acc.merge(:tag => m)
-      else acc.merge(m)
-      end
-    end
-
-    if !metadata.empty?
-      value.extend Metadata
-      value.metadata = metadata
-    end
-
-    value
+  def self.set(*elems)
+    Set.new(*elems)
   end
 
-
 end # EDN namespace

data/test/test_output_diff.rb

@@ -12,18 +12,47 @@ class EDNT_Test < Minitest::Test
   end
 
   def check_file(file, expected_output)
-    File.open(file) { |file|
-      assert_equal(expected_output, @parser.parse(file.read))
+    output = nil
+    File.open(file) { |source|
+      input = source.instance_of?(String) ? source : source.read
+
+      # test using parse() first
+      output = @parser.parse(input)
+      assert_equal(expected_output, output)
+
+      # now test setting the source and using read (although one-shot)
+      @parser.set_input(input)
+      output = @parser.read
+      assert_equal(expected_output, output)
     }
+    output
   end
 
-  def test_basic
+  # ========================================================================================
+  # tests start here
+  #
+  def test_false
+    check_file('test/false.edn', false)
+  end
+
+  def test_tru
+    check_file('test/true.edn', true)
+  end
+
+  def test_nil
+    check_file('test/nil.edn', nil)
+  end
+
+  def test_char
+    check_file('test/char.edn', "\f")
+  end
+
+  def test_string
+    check_file('test/string.edn', "abc\"➪\u8226")
+  end
 
-    assert_equal(false, @parser.parse('false'))
-    assert_equal(true, @parser.parse('true'))
-    assert_equal("a string", @parser.parse('"a string"'))
-    assert_equal(:"namespace.of.some.length/keyword-name", @parser.parse(':namespace.of.some.length/keyword-name'))
-    assert_equal(:'/', @parser.parse(':/'))
+  def test_keyword_with_namespace
+    check_file('test/keyword_with_namespace.edn', :"namespace.of.some.length/keyword-name")
   end
 
   def test_number
@@ -39,7 +68,7 @@ class EDNT_Test < Minitest::Test
   def test_keyword
 
     check_file('test/keyword.edn',
-               [:key1, :"key_2/adsd2", :key_3, :"key-4", :"key_5/asd-32_ee"]
+               [:key1, :"key_2/adsd2", :key_3, :"key-4", :"key_5/asd-32_ee", :"#/:a"]
               )
   end
 
@@ -47,7 +76,7 @@ class EDNT_Test < Minitest::Test
 
     check_file('test/values.edn',
                [false, true, nil, "this is a test", "this\tis\\only\ta\ttest\rof\"various\nescaped\\values",
-                "c", "\n", "\t",
+                ["c", "\n", "\t"],
                 "123➪456®789"]
               )
   end
@@ -89,8 +118,12 @@ class EDNT_Test < Minitest::Test
     check_file('test/symbol.edn',
                [
                  EDN::Type::Symbol.new("asymbol"),
+                 EDN::Type::Symbol.new(".asymbol"),
                  EDN::Type::Symbol.new("with'_a_'"),
                  EDN::Type::Symbol.new("with.123"),
+                 EDN::Type::Symbol.new("-with.123"),
+                 EDN::Type::Symbol.new("/"),
+                 EDN::Type::Symbol.new(">:FOuy/+"),
                ]
                )
 
@@ -112,13 +145,6 @@ class EDNT_Test < Minitest::Test
               )
   end
 
-  def test_read
-
-    # check read for using string
-    assert_equal({:a=>1, :b=>2}, @parser.parse('{:a 1 :b 2}'))
-
-  end
-
   def test_list
 
     check_file('test/list_1.edn',
@@ -143,15 +169,23 @@ class EDNT_Test < Minitest::Test
 
 
   def test_metadata
-    f = EDNT::read(File.open('test/metadata.edn').read)
-    assert_equal([98.6, 99.7], f)
+    f = check_file('test/metadata.edn', [98.6, 99.7])
     assert_equal({:doc=>"This is my vector", :rel=>:temps}, f.metadata)
+  end
 
-    f = EDNT::read(File.open('test/metadata2.edn').read)
-    assert_equal([1, 2], f)
+  def test_metadata2
+    f = check_file('test/metadata2.edn', [1, 2])
     assert_equal({:foo=>true, :tag=>EDN::Type::Symbol.new('String'), :bar=>2}, f.metadata)
   end
 
+  def test_metadata_in_vector
+    check_file('test/meta_in_vector.edn',
+               [ [ EDN::Type::Symbol.new('c'), :d, true ],
+                 DateTime.rfc3339("1390-09-07T21:27:03+00:00")
+               ]
+              )
+  end
+
   #
   # for testing tagged element #edn_turbo/test_tagged
   class Tagged
@@ -171,7 +205,7 @@ class EDNT_Test < Minitest::Test
       Tagged.new(data).to_s
     end
 
-    assert_equal([345, :a], @parser.parse('#edn_turbo/test_tagged { :item 345 :other :a }'))
+    check_file('test/tagged_elem.edn', [345, :a])
   end
 
   def test_operators
@@ -191,10 +225,6 @@ class EDNT_Test < Minitest::Test
                 EDN::Type::Symbol.new('='),
                 EDN::Type::Symbol.new('-'),
                 EDN::Type::Symbol.new('+'),
-#                [1, EDN::Type::Symbol.new('/'), 2],
-#                [3, EDN::Type::Symbol.new('/'), 4],
-                [5, EDN::Type::Symbol.new('/'), 6],
-                [7, EDN::Type::Symbol.new('/'), 8]
                ]
               )
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: edn_turbo
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Ed Porras
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-18 00:00:00.000000000 Z
+date: 2015-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: edn
@@ -75,12 +75,15 @@ files:
 - ext/edn_turbo/extconf.rb
 - ext/edn_turbo/main.cc
 - lib/edn_turbo.rb
+- lib/edn_turbo/constants.rb
 - lib/edn_turbo/edn_parser.rb
+- lib/edn_turbo/tags.rb
+- lib/edn_turbo/utils.rb
 - lib/edn_turbo/version.rb
 - test/test_output_diff.rb
 homepage: http://rubygems.org/gems/edn_turbo
 licenses:
-- ''
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []