kdl 0.1.2 → 1.0.1

data/lib/kdl/kdl.yy

@@ -4,59 +4,76 @@ class KDL::Parser
         STRING RAWSTRING
         INTEGER FLOAT TRUE FALSE NULL
         WS NEWLINE
+        LBRACE RBRACE
         LPAREN RPAREN
         EQUALS
         SEMICOLON
         EOF
         SLASHDASH
+        ESCLINE
 rule
   document  : nodes { KDL::Document.new(val[0]) }
             | linespaces { KDL::Document.new([])}
 
-  nodes     : none                      { [] }
-            | linespace_star node       { [val[1]] }
-            | linespace_star empty_node { [] }
-            | nodes node                { [*val[0], val[1]] }
-            | nodes empty_node          { val[0] }
-  node      : node_decl node_term                                 { val[0] }
-            | node_decl node_children node_term                   { val[0].tap { |x| x.children = val[1] } }
-            | node_decl empty_children node_term                   { val[0] }
-  node_decl : identifier                              { KDL::Node.new(val[0]) }
-            | node_decl WS value                      { val[0].tap { |x| x.arguments << val[2] } }
-            | node_decl WS SLASHDASH ws_star value    { val[0] }
-            | node_decl WS property                   { val[0].tap { |x| x.properties[val[2][0]] = val[2][1] } }
-            | node_decl WS SLASHDASH ws_star property { val[0] }
-  empty_node: SLASHDASH ws_star node
-  node_children: ws_star LPAREN nodes RPAREN { val[2] }
+nodes           : none                      { [] }
+                | linespaces node       { [val[1]] }
+                | linespaces empty_node { [] }
+                | nodes node                { [*val[0], val[1]] }
+                | nodes empty_node          { val[0] }
+  node          : untyped_node      { val[0] }
+                | type untyped_node { val[1].as_type(val[0], @type_parsers.fetch(val[0], nil)) }
+  untyped_node  : node_decl node_term                { val[0].tap { |x| x.children = [] } }
+                | node_decl node_children node_term  { val[0].tap { |x| x.children = val[1] } }
+                | node_decl empty_children node_term { val[0].tap { |x| x.children = [] } }
+  node_decl     : identifier                              { KDL::Node.new(val[0]) }
+                | node_decl ws value                      { val[0].tap { |x| x.arguments << val[2] } }
+                | node_decl ws SLASHDASH ws_star value    { val[0] }
+                | node_decl ws property                   { val[0].tap { |x| x.properties[val[2][0]] = val[2][1] } }
+                | node_decl ws SLASHDASH ws_star property { val[0] }
+  empty_node    : SLASHDASH ws_star node
+  node_children : ws_star LBRACE nodes RBRACE { val[2] }
+                | ws_star LBRACE linespaces RBRACE { [] }
   empty_children: SLASHDASH node_children
-                | WS empty_children
+                | ws empty_children
   node_term: linespaces | semicolon_term
   semicolon_term: SEMICOLON | SEMICOLON linespaces
 
-  identifier: IDENT  { val[0].value }
-            | STRING { val[0].value }
+  type : LPAREN identifier RPAREN { val[1] }
+
+  identifier: IDENT     { val[0].value }
+            | STRING    { val[0].value }
+            | RAWSTRING { val[0].value }
 
   property: identifier EQUALS value { [val[0], val[2]] }
 
-  value : STRING     { KDL::Value::String.new(val[0].value) }
-        | RAWSTRING  { KDL::Value::String.new(val[0].value) }
-        | INTEGER    { KDL::Value::Int.new(val[0].value) }
-        | FLOAT      { KDL::Value::Float.new(val[0].value) }
-        | boolean    { KDL::Value::Boolean.new(val[0]) }
-        | NULL       { KDL::Value::Null }
+  value : untyped_value
+        | type untyped_value { val[1].as_type(val[0], @type_parsers.fetch(val[0], nil)) }
+
+  untyped_value : STRING     { KDL::Value::String.new(val[0].value) }
+                | RAWSTRING  { KDL::Value::String.new(val[0].value) }
+                | INTEGER    { KDL::Value::Int.new(val[0].value) }
+                | FLOAT      { KDL::Value::Float.new(val[0].value, format: val[0].meta[:format]) }
+                | boolean    { KDL::Value::Boolean.new(val[0]) }
+                | NULL       { KDL::Value::Null }
 
   boolean : TRUE  { true }
           | FALSE { false }
 
-  ws_star: none | WS
+  ws: WS | ESCLINE | WS ESCLINE | ESCLINE WS | WS ESCLINE WS
+  ws_star: none | ws
   linespace: WS | NEWLINE | EOF
   linespaces: linespace | linespaces linespace
-  linespace_star: none | linespaces
 
   none: { nil }
 
 ---- inner
-  def parse(str)
+
+  def parse(str, options = {})
+    if options.fetch(:parse_types, true)
+      @type_parsers = ::KDL::Types::MAPPING.merge(options.fetch(:type_parsers, {}))
+    else
+      @type_parsers = {}
+    end
     @tokenizer = ::KDL::Tokenizer.new(str)
     do_parse
   end

data/lib/kdl/node.rb

@@ -1,31 +1,31 @@
 module KDL
   class Node
-    attr_accessor :name, :arguments, :properties, :children
+    attr_accessor :name, :arguments, :properties, :children, :type
 
-    def initialize(name, arguments = [], properties = {}, children = [])
+    def initialize(name, arguments = [], properties = {}, children = [], type: nil)
       @name = name
       @arguments = arguments
       @properties = properties
       @children = children
+      @type = type
     end
 
     def to_s(level = 0)
       indent = '    ' * level
-      s = "#{indent}#{name}"
+      s = "#{indent}#{type ? "(#{id_to_s type})" : ''}#{id_to_s name}"
       unless arguments.empty?
         s += " #{arguments.map(&:to_s).join(' ')}"
       end
       unless properties.empty?
-        s += " #{properties.map { |k, v| "#{k}=#{v}" }.join(' ')}"
+        s += " #{properties.map { |k, v| "#{id_to_s k}=#{v}" }.join(' ')}"
       end
       unless children.empty?
         s += " {\n"
-        s += children.map { |c| c.to_s(level + 1) }.join("\n")
-        s += "\n#{indent}}"
+        s += children.map { |c| "#{c.to_s(level + 1)}\n" }.join("\n")
+        s += "#{indent}}"
       end
       s
     end
-    alias inspect to_s
 
     def ==(other)
       return false unless other.is_a?(Node)
@@ -35,5 +35,28 @@ module KDL
       properties == other.properties &&
       children   == other.children
     end
+
+    def as_type(type, parser = nil)
+      if parser.nil?
+        @type = type
+        self
+      else
+        result = parser.call(self, type)
+
+        return self.as_type(type) if result.nil?
+
+        unless result.is_a?(::KDL::Node)
+          raise ArgumentError, "expected parser to return an instance of ::KDL::Node, got `#{result.class}'"
+        end
+
+        result
+      end
+    end
+
+    private
+
+    def id_to_s(id)
+      StringDumper.stringify_identifier(id)
+    end
   end
 end

data/lib/kdl/string_dumper.rb

@@ -0,0 +1,45 @@
+module KDL
+  module StringDumper
+    class << self
+      def call(string)
+        %("#{string.each_char.map { |char| escape(char) }.join}")
+      end
+
+      def stringify_identifier(ident)
+        if bare_identifier?(ident)
+          ident
+        else
+          call(ident)
+        end
+      end
+
+      private
+
+      def print?(char)
+        ' ' <= char && char <= '\x7e'
+      end
+
+      def escape(char)
+        case char
+        when "\n" then '\n'
+        when "\r" then '\r'
+        when "\t" then '\t'
+        when '\\' then '\\\\'
+        when '"' then '\"'
+        when "\b" then '\b'
+        when "\f" then '\f'
+        else char
+        end
+      end
+
+      def unicode_escape(char)
+        "\\u{#{char.codepoints.first.to_s(16)}}"
+      end
+
+      def bare_identifier?(name)
+        escape_chars = '\\\/(){}<>;\[\]=,"'
+        name =~ /^([^0-9\-+\s#{escape_chars}][^\s#{escape_chars}]*|[\-+](?!true|false|null)[^0-9\s#{escape_chars}][^\s#{escape_chars}]*)$/
+      end
+    end
+  end
+end

data/lib/kdl/tokenizer.rb

@@ -1,3 +1,5 @@
+require 'bigdecimal'
+
 module KDL
   class Tokenizer
     class Error < StandardError
@@ -7,13 +9,14 @@ module KDL
     end
 
     class Token
-      attr_reader :type, :value, :line, :column
+      attr_reader :type, :value, :line, :column, :meta
 
-      def initialize(type, value, line, column)
+      def initialize(type, value, line, column, meta = {})
         @type = type
         @value = value
         @line = line
         @column = column
+        @meta = meta
       end
 
       def ==(other)
@@ -31,8 +34,8 @@ module KDL
     attr_reader :index
 
     SYMBOLS = {
-      '{' => :LPAREN,
-      '}' => :RPAREN,
+      '{' => :LBRACE,
+      '}' => :RBRACE,
       '=' => :EQUALS,
       '＝' => :EQUALS,
       ';' => :SEMICOLON
@@ -46,10 +49,13 @@ module KDL
 
     NEWLINES = ["\u000A", "\u0085", "\u000C", "\u2028", "\u2029"]
 
-    NON_IDENTIFIER_CHARS = Regexp.escape "#{SYMBOLS.keys.join('')}\\<>[]\","
+    NON_IDENTIFIER_CHARS = Regexp.escape "#{SYMBOLS.keys.join('')}()/\\<>[]\","
     IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}\x0-\x20]/
     INITIAL_IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}0-9\x0-\x20]/
 
+    ALLOWED_IN_TYPE = [:ident, :string, :rawstring]
+    NOT_ALLOWED_AFTER_TYPE = [:single_line_comment, :multi_line_comment]
+
     def initialize(str, start = 0)
       @str = str
       @context = nil
@@ -60,6 +66,8 @@ module KDL
       @previous_context = nil
       @line = 1
       @column = 1
+      @type_context = false
+      @last_token = nil
     end
 
     def next_token
@@ -117,15 +125,13 @@ module KDL
             end
           when '\\'
             t = Tokenizer.new(@str, @index + 1)
-            la = t.next_token[0]
-            if la == :NEWLINE
-              @index = t.index
-              new_line
-            elsif la == :WS && (lan = t.next_token[0]) == :NEWLINE
+            la = t.next_token
+            if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF)
               @index = t.index
               new_line
+              return token(:ESCLINE, "\\#{la[1].value}")
             else
-              raise_error "Unexpected '\\'"
+              raise_error "Unexpected '\\' (#{la[0]})"
             end
           when *SYMBOLS.keys
             return token(SYMBOLS[c], c).tap { traverse(1) }
@@ -174,6 +180,12 @@ module KDL
             self.context = :ident
             @buffer = c
             traverse(1)
+          when '('
+            @type_context = true
+            return token(:LPAREN, c).tap { traverse(1) }
+          when ')'
+            @type_context = false
+            return token(:RPAREN, c).tap { traverse(1) }
           else
             raise_error "Unexpected character #{c.inspect}"
           end
@@ -279,18 +291,6 @@ module KDL
           if WHITEPACE.include?(c)
             traverse(1)
             @buffer += c
-          elsif c == "\\"
-            t = Tokenizer.new(@str, @index + 1)
-            la = t.next_token[0]
-            if la == :NEWLINE
-              @index = t.index
-              new_line
-            elsif (la == :WS && (lan = t.next_token[0]) == :NEWLINE)
-              @index = t.index
-              new_line
-            else
-              raise_error "Unexpected '\\'"
-            end
           elsif c == "/" && @str[@index + 1] == '*'
             self.context = :multi_line_comment
             @comment_nesting = 1
@@ -304,8 +304,8 @@ module KDL
 
     private
 
-    def token(type, value)
-      [type, Token.new(type, value, @line_at_start, @column_at_start)]
+    def token(type, value, **meta)
+      @last_token = [type, Token.new(type, value, @line_at_start, @column_at_start, meta)]
     end
 
     def traverse(n = 1)
@@ -323,6 +323,11 @@ module KDL
     end
 
     def context=(val)
+      if @type_context && !ALLOWED_IN_TYPE.include?(val)
+        raise_error "#{val} context not allowed in type declaration"
+      elsif @last_token && @last_token[0] == :RPAREN && NOT_ALLOWED_AFTER_TYPE.include?(val)
+        raise_error 'Comments are not allowed after a type declaration'
+      end
       @previous_context = @context
       @context = val
     end
@@ -333,18 +338,35 @@ module KDL
     end
 
     def parse_decimal(s)
-      return token(:FLOAT, Float(munch_underscores(s))) if s =~ /[.eE]/
-      token(:INTEGER, Integer(munch_underscores(s), 10))
+      return parse_float(s) if s =~ /[.E]/i
+
+      token(:INTEGER, Integer(munch_underscores(s), 10), format: '%d')
+    end
+
+    def parse_float(s)
+      match, _, fraction, exponent = *s.match(/^([-+]?[\d_]+)(?:\.([\d_]+))?(?:[eE]([-+]?[\d_]+))?$/)
+      raise_error "Invalid floating point value #{s}" if match.nil?
+
+      s = munch_underscores(s)
+
+      decimals = fraction.nil? ? 0 : fraction.size
+      value = Float(s)
+      scientific = value.abs >= 100 || (exponent && exponent.to_i.abs >= 2)
+      if value.infinite? || (value.zero? && exponent.to_i < 0)
+        token(:FLOAT, BigDecimal(s))
+      else
+        token(:FLOAT, value, format: scientific ? "%.#{decimals}E" : nil)
+      end
     end
-    
+
     def parse_hexadecimal(s)
       token(:INTEGER, Integer(munch_underscores(s), 16))
     end
-    
+
     def parse_octal(s)
       token(:INTEGER, Integer(munch_underscores(s), 8))
     end
-    
+
     def parse_binary(s)
       token(:INTEGER, Integer(munch_underscores(s), 2))
     end

data/lib/kdl/types/base64.rb

@@ -0,0 +1,15 @@
+require 'base64'
+
+module KDL
+  module Types
+    class Base64 < Value
+      def self.call(value, type = 'base64')
+        return nil unless value.is_a? ::KDL::Value::String
+
+        data = ::Base64.decode64(value.value)
+        new(data, type: type)
+      end
+    end
+    MAPPING['base64'] = Base64
+  end
+end