edn_turbo 0.5.3 → 0.5.4

data/ext/edn_turbo/util.cc

@@ -0,0 +1,240 @@
+#include <iostream>
+#include <string>
+#include <sstream>
+#include <limits>
+
+#include <ruby/ruby.h>
+#include <ruby/encoding.h>
+
+#include "util.h"
+#include "util_unicode.h"
+
+namespace edn
+{
+    //
+    // used to determine max number of chars in string value of a type
+    template <typename T>
+    static std::size_t get_max_chars(T)
+    {
+        std::stringstream s;
+        s << std::fixed << std::numeric_limits<T>::max();
+        return s.str().length();
+    }
+
+    static const std::size_t LL_max_chars = get_max_chars<>((long) 1);
+    static const std::size_t LD_max_chars = get_max_chars<>((double) 1);
+
+    //
+    // throw runtime error
+    static void throw_error(int error)
+    {
+        if (error == 0)
+            return;
+
+        VALUE err = rb_errinfo();
+        rb_raise(CLASS_OF(err), "%s", RSTRING_PTR(rb_obj_as_string(err)));
+    }
+
+    // =================================================================
+    // work-around for idiotic rb_protect convention in order to avoid
+    // using ruby/rice
+    //
+    typedef VALUE (edn_rb_f_type)( VALUE arg );
+
+    // we're using at most 2 args
+    struct prot_args {
+        prot_args(VALUE r, ID m) :
+            receiver(r), method(m), count(0) {
+        }
+        prot_args(VALUE r, ID m, VALUE arg) :
+            receiver(r), method(m), count(1) {
+            args[0] = arg;
+        }
+        prot_args(VALUE r, ID m, VALUE arg1, VALUE arg2) :
+            receiver(r), method(m), count(2) {
+            args[0] = arg1;
+            args[1] = arg2;
+        }
+
+        VALUE call() const {
+            return ((count == 0) ?
+                    rb_funcall( receiver, method, 0 ) :
+                    rb_funcall2( receiver, method, count, args ));
+        }
+
+    private:
+        VALUE receiver;
+        ID method;
+        int count;
+        VALUE args[2];
+    };
+
+    // this allows us to wrap with rb_protect()
+    static inline VALUE edn_wrap_funcall2( VALUE arg ) {
+        const prot_args* a = reinterpret_cast<const prot_args*>(arg);
+        if (a)
+            return a->call();
+        return Qnil;
+    }
+
+    static inline VALUE edn_prot_rb_funcall( edn_rb_f_type func, VALUE args ) {
+        int error;
+        VALUE s = rb_protect( func, args, &error );
+        if (error) throw_error(error);
+        return s;
+    }
+
+    static inline VALUE edn_prot_rb_new_str(const char* str) {
+        int error;
+        VALUE s = rb_protect( reinterpret_cast<VALUE (*)(VALUE)>(rb_str_new_cstr),
+                              reinterpret_cast<VALUE>(str), &error );
+        if (error) throw_error(error);
+        return s;
+    }
+
+    static inline VALUE edn_rb_enc_associate_utf8(VALUE str) {
+        return rb_enc_associate(str, rb_utf8_encoding() );
+    }
+
+    // =================================================================
+    // utils
+    namespace util
+    {
+        // utility method to convert a primitive in string form to a
+        // ruby type
+        template <class T>
+        static inline T buftotype(const char* p, std::size_t len) {
+            T val;
+            std::string buf;
+            buf.append(p, len);
+            std::istringstream(buf) >> val;
+            return val;
+        }
+
+        //
+        // convert to int.. if string rep has more digits than long can
+        // hold, call into ruby to get a big num
+        VALUE integer_to_ruby(const char* str, std::size_t len)
+        {
+            // if something bigger than a long is needed, call into
+            // ruby side to get the correct value
+            if (str[len-1] == 'M' || len >= LL_max_chars)
+            {
+                std::string buf(str, len);
+                VALUE vs = edn_prot_rb_new_str(buf.c_str());
+                prot_args args(vs, RUBY_STRING_TO_I_METHOD);
+                return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+            }
+
+            return LONG2NUM(buftotype<long>(str, len));
+        }
+
+        //
+        // as above.. TODO: check exponential..
+        VALUE float_to_ruby(const char* str, std::size_t len)
+        {
+            // if big decimal is needed, call into ruby side to get
+            // the correct value
+            if (str[len-1] == 'M' || len >= LD_max_chars)
+            {
+                std::string buf(str, len);
+                VALUE vs = edn_prot_rb_new_str(buf.c_str());
+
+                if (str[len-1] == 'M') {
+                    return call_module_fn(rb_mEDN, EDN_MAKE_BIG_DECIMAL_METHOD, vs);
+                }
+
+                prot_args args(vs, RUBY_STRING_TO_F_METHOD);
+                return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+            }
+
+            return rb_float_new(buftotype<double>(str, len));
+        }
+
+
+        //
+        // read from a StringIO - expensive!!!
+        //
+        VALUE ruby_io_read(VALUE io)
+        {
+            prot_args args(io, RUBY_READ_METHOD);
+            return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+        }
+
+        //
+        // copies the string data, unescaping any present values that need to be replaced
+        //
+        bool parse_byte_stream(const char *p_start, const char *p_end, VALUE& v_utf8, bool encode)
+        {
+            if (p_end > p_start) {
+                std::string buf;
+
+                if (encode) {
+                    if (!util::unicode::to_utf8(p_start, (uint32_t) (p_end - p_start), buf))
+                        return false;
+                }
+                else {
+                    buf.append(p_start, p_end - p_start);
+                }
+
+                // utf-8 encode
+                VALUE vs = edn_prot_rb_new_str(buf.c_str());
+                int error;
+                v_utf8 = rb_protect( edn_rb_enc_associate_utf8, vs, &error);
+                if (error) throw_error(error);
+                return true;
+            } else if (p_end == p_start) {
+                v_utf8 = rb_str_new("", 0);
+                return true;
+            }
+
+            return false;
+        }
+
+        //
+        // handles things like \c, \newline
+        //
+        bool parse_escaped_char(const char *p, const char *pe, VALUE& v)
+        {
+            std::string buf;
+            std::size_t len = pe - p;
+            buf.append(p, len);
+
+            if (len > 1) {
+                if      (buf == "newline") buf = '\n';
+                else if (buf == "tab") buf = '\t';
+                else if (buf == "return") buf = '\r';
+                else if (buf == "space") buf = ' ';
+                else if (buf == "formfeed") buf = '\f';
+                else if (buf == "backspace") buf = '\b';
+                // TODO: is this supported?
+                else if (buf == "verticaltab") buf = '\v';
+                else return false;
+            }
+
+            v = edn_prot_rb_new_str( buf.c_str() );
+            return true;
+        }
+
+
+        //
+        // get a set representation from the ruby side. See edn_turbo.rb
+        VALUE call_module_fn(VALUE module, ID method)
+        {
+            prot_args args(module, method);
+            return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+        }
+
+        VALUE call_module_fn(VALUE module, ID method, VALUE value)
+        {
+            prot_args args(module, method, value);
+            return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+        }
+
+        VALUE call_module_fn(VALUE module, ID method, VALUE name, VALUE data)
+        {
+            prot_args args(module, method, name, data);
+            return edn_prot_rb_funcall( edn_wrap_funcall2, reinterpret_cast<VALUE>(&args) );
+        }
+    }
+}

data/ext/edn_turbo/util.h

@@ -0,0 +1,39 @@
+#pragma once
+
+namespace edn
+{
+    extern VALUE rb_mEDN;
+    extern VALUE rb_mEDNT;
+
+    extern VALUE EDN_MAKE_SYMBOL_METHOD;
+    extern VALUE EDN_MAKE_LIST_METHOD;
+    extern VALUE EDN_MAKE_SET_METHOD;
+    extern VALUE EDN_MAKE_BIG_DECIMAL_METHOD;
+    extern VALUE EDN_TAGGED_ELEM_METHOD;
+    extern VALUE EDN_EOF_CONST;
+
+    extern VALUE EDNT_EXTENDED_VALUE_METHOD;
+
+    extern VALUE RUBY_STRING_TO_I_METHOD;
+    extern VALUE RUBY_STRING_TO_F_METHOD;
+    extern VALUE RUBY_READ_METHOD;
+
+    namespace util
+    {
+        // defined in edn_parser_util.cc
+        VALUE integer_to_ruby(const char* str, std::size_t len);
+        VALUE float_to_ruby  (const char* str, std::size_t len);
+
+        VALUE ruby_io_read(VALUE io);
+
+        bool parse_byte_stream (const char *p, const char *pe, VALUE& rslt, bool encode);
+        bool parse_escaped_char(const char *p, const char *pe, VALUE& rslt);
+
+        VALUE call_module_fn(VALUE module, ID method);
+        VALUE call_module_fn(VALUE module, ID method, VALUE value);
+        VALUE call_module_fn(VALUE module, ID method, VALUE value1, VALUE value2);
+
+        // edn_parser_util_unicode.cc
+        bool to_utf8(const char *s, uint32_t len, std::string& rslt);
+    }
+}

data/ext/edn_turbo/util_unicode.cc

@@ -0,0 +1,36 @@
+#include <string>
+
+//
+// needed to define this in its own file because icu and ruby have
+// differing definitions for Uchar and the compiler complains
+//
+#include <unicode/utypes.h>
+#include <unicode/ustring.h>
+#include <unicode/ucnv.h>
+#include <unicode/unistr.h>
+
+#include "util_unicode.h"
+
+namespace edn
+{
+    namespace util
+    {
+        namespace unicode
+        {
+            //
+            // unescapes any values that need to be replaced, saves it to utf8
+            //
+            bool to_utf8(const char *s, uint32_t len, std::string& rslt)
+            {
+                icu::UnicodeString ustr(s, len);
+
+                if (ustr.isBogus()) {
+                    return false;
+                }
+
+                ustr.unescape().toUTF8String(rslt);
+                return true;
+            }
+        }
+    }
+}

data/ext/edn_turbo/util_unicode.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <string>
+
+namespace edn
+{
+    namespace util
+    {
+        namespace unicode
+        {
+            bool to_utf8(const char *s, uint32_t len, std::string& rslt);
+        }
+    }
+}

data/lib/edn_turbo/edn_parser.rb

@@ -1,18 +1,21 @@
+# frozen_string_literal: true
+
+#
+# methods specific to edn_turbo
 module EDNT
   # Bind the given meta to the value.
   def self.extend_for_meta(value, ext_meta)
-
     meta = ext_meta
 
     metadata = meta.reduce({}) do |acc, m|
       case m
       when Symbol then acc.merge(m => true)
-      when EDN::Type::Symbol then acc.merge(:tag => m)
+      when EDN::Type::Symbol then acc.merge(tag: m)
       else acc.merge(m)
       end
     end
 
-    if !metadata.empty?
+    unless metadata.empty?
       value.extend EDN::Metadata
       value.metadata = metadata
     end

data/lib/edn_turbo/version.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 module EDNT
-  VERSION = '0.5.3'
-  RELEASE_DATE = '2017-08-16'
+  VERSION = '0.5.4'
+  RELEASE_DATE = '2017-12-29'
 end

data/lib/edn_turbo.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'edn'
 require 'edn_turbo/version'
 require 'edn_turbo/edn_parser'

data/test/test_output_diff.rb

@@ -1,12 +1,11 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
-$LOAD_PATH << File.expand_path( File.dirname(__FILE__) + '/../lib' )
+$LOAD_PATH << File.expand_path(File.dirname(__FILE__) + '/../lib')
 require 'minitest/autorun'
 require 'edn_turbo'
 require 'date'
 
 class EDNT_Test < Minitest::Test
-
   def setup
     @parser = EDNT::Parser.new
   end
@@ -47,7 +46,7 @@ class EDNT_Test < Minitest::Test
     output
   end
 
-  # ========================================================================================
+  # ============================================================================
   # tests start here
   #
   def test_false
@@ -95,8 +94,8 @@ class EDNT_Test < Minitest::Test
 
     check_file('test/values.edn',
                [false, true, nil, "this is a test", "this\tis\\only\ta\ttest\rof\"various\nescaped\\values",
-                ["c", "\n", "\t"],
-                "123➪456®789"]
+                ['c', "\n", "\t"],
+                '123➪456®789']
               )
   end
 
@@ -104,103 +103,94 @@ class EDNT_Test < Minitest::Test
 
     check_file('test/inst.edn',
                [
-                 DateTime.rfc3339("1985-04-12T23:20:50.52Z"),
-                 DateTime.rfc3339("1996-12-19T16:39:57-08:00"),
-                 DateTime.rfc3339("1990-12-31T23:59:60Z"),
-                 DateTime.rfc3339("1990-12-31T15:59:60-08:00"),
-                 DateTime.rfc3339("1937-01-01T12:00:27.87+00:20")
+                 DateTime.rfc3339('1985-04-12T23:20:50.52Z'),
+                 DateTime.rfc3339('1996-12-19T16:39:57-08:00'),
+                 DateTime.rfc3339('1990-12-31T23:59:60Z'),
+                 DateTime.rfc3339('1990-12-31T15:59:60-08:00'),
+                 DateTime.rfc3339('1937-01-01T12:00:27.87+00:20')
                ]
               )
   end
 
   def test_builtin_tagged_uuid
 
-    check_file('test/uuid.edn', "f81d4fae-7dec-11d0-a765-00a0c91e6bf6")
+    check_file('test/uuid.edn', 'f81d4fae-7dec-11d0-a765-00a0c91e6bf6')
 
   end
 
   def test_sets
-
     check_file('test/set.edn',
                [ Set.new(),
                  Set.new([1, Set.new([:abc])]),
                  Set.new([1]),
-                 Set.new([1, "abc" ]),
-                 Set.new([1, 2, 3, [4, 5], "abc", 23.3])
+                 Set.new([1, 'abc' ]),
+                 Set.new([1, 2, 3, [4, 5], 'abc', 23.3])
                ]
               )
-
   end
 
   def test_symbol
-
     check_file('test/symbol.edn',
                [
-                 EDN::Type::Symbol.new("asymbol"),
-                 EDN::Type::Symbol.new(".asymbol"),
+                 EDN::Type::Symbol.new('asymbol'),
+                 EDN::Type::Symbol.new('.asymbol'),
                  EDN::Type::Symbol.new("with'_a_'"),
-                 EDN::Type::Symbol.new("with.123"),
-                 EDN::Type::Symbol.new("-with.123"),
-                 EDN::Type::Symbol.new("/"),
-                 EDN::Type::Symbol.new(">:FOuy/+"),
+                 EDN::Type::Symbol.new('with.123'),
+                 EDN::Type::Symbol.new('-with.123'),
+                 EDN::Type::Symbol.new('/'),
+                 EDN::Type::Symbol.new('>:FOuy/+'),
                ]
                )
-
   end
 
   def test_unicode
-
     check_file('test/unicode.edn',
                [:text,
                 "Page \u0018, October 2009 TechTIPS",
-                "This should be an unfilled star: ☆"]
+                'This should be an unfilled star: ☆']
               )
   end
 
   def test_vector
-
     check_file('test/vector_1.edn',
-               [true, true, 34, [true, nil, "añ", "", :test, [3213.23]]]
+               [true, true, 34, [true, nil, 'añ', '', :test, [3213.23]]]
               )
   end
 
   def test_list
-
     check_file('test/list_1.edn',
-               [22, 3312, "dss", {:powerpuff=>[:buttercup, :bubbles, :blossom]}]
+               [22, 3312, 'dss', {:powerpuff=>[:buttercup, :bubbles, :blossom]}]
               )
-
   end
 
   def test_map
-
     check_file('test/map1.edn',
-               {:key_a1=>true,:key_a2=>false,:key_a3=>[1, 2, 3, "test string", nil, {1=>2}],
-                :key_a4=>{:key_a31=>23, :key_a32=>24.4},"string_key"=>:kval,
-                :embedded=>[true, {:c2g_md5=>"2bbee1cd3045710db6fec432b00d1e0c"}],
+               {:key_a1=>true,:key_a2=>false,:key_a3=>[1, 2, 3, 'test string', nil, {1=>2}],
+                :key_a4=>{:key_a31=>23, :key_a32=>24.4},'string_key'=>:kval,
+                :embedded=>[true, {:c2g_md5=>'2bbee1cd3045710db6fec432b00d1e0c'}],
                 2=>{:a=>:b}}
               )
 
     check_file('test/map2.edn',
-               {:int=>1, :string=>"hello", :char=>"a", :array=>[0, 1], :hash=>{:key=>"value"}}
+               {int: 1, string: 'hello', char: 'a', array: [0, 1], hash: {key: 'value'}}
               )
   end
 
 
   def test_metadata
     f = check_file('test/metadata.edn', [98.6, 99.7])
-    assert_equal({:doc=>"This is my vector", :rel=>:temps}, f.metadata)
+    assert_equal({:doc=>'This is my vector', rel: :temps}, f.metadata)
   end
 
   def test_metadata2
     f = check_file('test/metadata2.edn', [1, 2])
-    assert_equal({:foo=>true, :tag=>EDN::Type::Symbol.new('String'), :bar=>2}, f.metadata)
+    assert_equal({ foo: true, tag: EDN::Type::Symbol.new('String'), bar: 2 }, f.metadata)
   end
 
   def test_metadata_in_vector
     check_file('test/meta_in_vector.edn',
                [ [ EDN::Type::Symbol.new('c'), :d, true ],
-                 DateTime.rfc3339("1390-09-07T21:27:03+00:00")
+                 DateTime.rfc3339('1390-09-07T21:27:03+00:00')
                ]
               )
   end
@@ -220,7 +210,7 @@ class EDNT_Test < Minitest::Test
 
   def test_tagged_elem
 
-    EDN.register("edn_turbo/test_tagged") do |data|
+    EDN.register('edn_turbo/test_tagged') do |data|
       Tagged.new(data).to_s
     end
 
@@ -257,7 +247,7 @@ class EDNT_Test < Minitest::Test
   end
 
   REF_MAP3_DATA = {:meta=>{:data_format_version=>304,
-                           :filename=>"test/colorspan.pdf",
+                           :filename=>'test/colorspan.pdf',
                            :is_ok=>true,
                            :font_engine_ok=>true,
                            :pdf_ver_major=>1,
@@ -388,7 +378,7 @@ class EDNT_Test < Minitest::Test
   end
 
   def test_ruby_file_io
-    File.open("test/true.edn", "r") do |f|
+    File.open('test/true.edn', 'r') do |f|
       # now test setting the source and using read (although one-shot)
       @parser.set_input(f)
       output = @parser.read
@@ -397,12 +387,12 @@ class EDNT_Test < Minitest::Test
   end
 
   def test_ruby_file_io_long
-      File.open("test/map3.edn", "r") do |f|
-        # now test setting the source and using read (although one-shot)
-        @parser.set_input(f)
-        output = @parser.read
-        assert_equal(REF_MAP3_DATA, output)
-      end
+    File.open('test/map3.edn', 'r') do |f|
+      # now test setting the source and using read (although one-shot)
+      @parser.set_input(f)
+      output = @parser.read
+      assert_equal(REF_MAP3_DATA, output)
+    end
   end
 
   def test_ruby_string_io
@@ -415,11 +405,10 @@ class EDNT_Test < Minitest::Test
 
   def test_init_arg
     # test passing IO to constructor
-    File.open("test/map3.edn", "r") do |f|
+    File.open('test/map3.edn', 'r') do |f|
       p = EDNT::Parser.new(f)
       output = p.read
       assert_equal(REF_MAP3_DATA, output)
     end
   end
-  
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: edn_turbo
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - Ed Porras
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-16 00:00:00.000000000 Z
+date: 2017-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: edn
@@ -83,13 +83,15 @@ files:
 - bin/ppedn-ruby
 - ext/edn_turbo/depend
 - ext/edn_turbo/edn_parser.cc
-- ext/edn_turbo/edn_parser.h
 - ext/edn_turbo/edn_parser.rl
-- ext/edn_turbo/edn_parser_util.cc
-- ext/edn_turbo/edn_parser_util.h
-- ext/edn_turbo/edn_parser_util_unicode.cc
 - ext/edn_turbo/extconf.rb
 - ext/edn_turbo/main.cc
+- ext/edn_turbo/parser.h
+- ext/edn_turbo/parser_def.cc
+- ext/edn_turbo/util.cc
+- ext/edn_turbo/util.h
+- ext/edn_turbo/util_unicode.cc
+- ext/edn_turbo/util_unicode.h
 - lib/edn_turbo.rb
 - lib/edn_turbo/edn_parser.rb
 - lib/edn_turbo/version.rb
@@ -114,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 3
 summary: Read EDN files