kdl 0.1.1 → 1.0.0

data/lib/kdl/kdl.yy

@@ -4,6 +4,7 @@ class KDL::Parser
         STRING RAWSTRING
         INTEGER FLOAT TRUE FALSE NULL
         WS NEWLINE
+        LBRACE RBRACE
         LPAREN RPAREN
         EQUALS
         SEMICOLON
@@ -13,37 +14,46 @@ rule
   document  : nodes { KDL::Document.new(val[0]) }
             | linespaces { KDL::Document.new([])}
 
-  nodes     : none                      { [] }
-            | linespace_star node       { [val[1]] }
-            | linespace_star empty_node { [] }
-            | nodes node                { [*val[0], val[1]] }
-            | nodes empty_node          { val[0] }
-  node      : node_decl node_term                                 { val[0] }
-            | node_decl node_children node_term                   { val[0].tap { |x| x.children = val[1] } }
-            | node_decl empty_children node_term                   { val[0] }
-  node_decl : identifier                              { KDL::Node.new(val[0]) }
-            | node_decl WS value                      { val[0].tap { |x| x.arguments << val[2] } }
-            | node_decl WS SLASHDASH ws_star value    { val[0] }
-            | node_decl WS property                   { val[0].tap { |x| x.properties[val[2][0]] = val[2][1] } }
-            | node_decl WS SLASHDASH ws_star property { val[0] }
-  empty_node: SLASHDASH ws_star node
-  node_children: ws_star LPAREN nodes RPAREN { val[2] }
+nodes           : none                      { [] }
+                | linespace_star node       { [val[1]] }
+                | linespace_star empty_node { [] }
+                | nodes node                { [*val[0], val[1]] }
+                | nodes empty_node          { val[0] }
+  node          : untyped_node      { val[0] }
+                | type untyped_node { val[1].as_type(val[0], @type_parsers.fetch(val[0], nil)) }
+  untyped_node  : node_decl node_term                { val[0].tap { |x| x.children = nil } }
+                | node_decl node_children node_term  { val[0].tap { |x| x.children = val[1] } }
+                | node_decl empty_children node_term { val[0].tap { |x| x.children = nil } }
+  node_decl     : identifier                              { KDL::Node.new(val[0]) }
+                | node_decl WS value                      { val[0].tap { |x| x.arguments << val[2] } }
+                | node_decl WS SLASHDASH ws_star value    { val[0] }
+                | node_decl WS property                   { val[0].tap { |x| x.properties[val[2][0]] = val[2][1] } }
+                | node_decl WS SLASHDASH ws_star property { val[0] }
+  empty_node    : SLASHDASH ws_star node
+  node_children : ws_star LBRACE nodes RBRACE { val[2] }
+                | ws_star LBRACE linespace_star RBRACE { [] }
   empty_children: SLASHDASH node_children
                 | WS empty_children
   node_term: linespaces | semicolon_term
   semicolon_term: SEMICOLON | SEMICOLON linespaces
 
-  identifier: IDENT  { val[0] }
-            | STRING { val[0] }
+  type : LPAREN identifier RPAREN { val[1] }
+
+  identifier: IDENT     { val[0].value }
+            | STRING    { val[0].value }
+            | RAWSTRING { val[0].value }
 
   property: identifier EQUALS value { [val[0], val[2]] }
 
-  value : STRING     { KDL::Value::String.new(val[0]) }
-        | RAWSTRING  { KDL::Value::String.new(val[0]) }
-        | INTEGER    { KDL::Value::Int.new(val[0]) }
-        | FLOAT      { KDL::Value::Float.new(val[0]) }
-        | boolean    { KDL::Value::Boolean.new(val[0]) }
-        | NULL       { KDL::Value::Null }
+  value : untyped_value
+        | type untyped_value { val[1].as_type(val[0], @type_parsers.fetch(val[0], nil)) }
+
+  untyped_value : STRING     { KDL::Value::String.new(val[0].value) }
+                | RAWSTRING  { KDL::Value::String.new(val[0].value) }
+                | INTEGER    { KDL::Value::Int.new(val[0].value) }
+                | FLOAT      { KDL::Value::Float.new(val[0].value, format: val[0].meta[:format]) }
+                | boolean    { KDL::Value::Boolean.new(val[0]) }
+                | NULL       { KDL::Value::Null }
 
   boolean : TRUE  { true }
           | FALSE { false }
@@ -56,7 +66,13 @@ rule
   none: { nil }
 
 ---- inner
-  def parse(str)
+
+  def parse(str, options = {})
+    if options.fetch(:parse_types, true)
+      @type_parsers = ::KDL::Types::MAPPING.merge(options.fetch(:type_parsers, {}))
+    else
+      @type_parsers = {}
+    end
     @tokenizer = ::KDL::Tokenizer.new(str)
     do_parse
   end

data/lib/kdl/node.rb

@@ -1,31 +1,33 @@
 module KDL
   class Node
-    attr_accessor :name, :arguments, :properties, :children
+    attr_accessor :name, :arguments, :properties, :children, :type
 
-    def initialize(name, arguments = [], properties = {}, children = [])
+    def initialize(name, arguments = [], properties = {}, children = nil, type: nil)
       @name = name
       @arguments = arguments
       @properties = properties
       @children = children
+      @type = type
     end
 
     def to_s(level = 0)
       indent = '    ' * level
-      s = "#{indent}#{name}"
+      s = "#{indent}#{type ? "(#{id_to_s type})" : ''}#{id_to_s name}"
       unless arguments.empty?
         s += " #{arguments.map(&:to_s).join(' ')}"
       end
       unless properties.empty?
-        s += " #{properties.map { |k, v| "#{k}=#{v}" }.join(' ')}"
+        s += " #{properties.map { |k, v| "#{id_to_s k}=#{v}" }.join(' ')}"
       end
-      unless children.empty?
+      unless children.nil?
         s += " {\n"
-        s += children.map { |c| c.to_s(level + 1) }.join("\n")
-        s += "\n#{indent}}"
+        unless children.empty?
+          s += children.map { |c| "#{c.to_s(level + 1)}\n" }.join("\n")
+        end
+        s += "#{indent}}"
       end
       s
     end
-    alias inspect to_s
 
     def ==(other)
       return false unless other.is_a?(Node)
@@ -35,5 +37,28 @@ module KDL
       properties == other.properties &&
       children   == other.children
     end
+
+    def as_type(type, parser = nil)
+      if parser.nil?
+        @type = type
+        self
+      else
+        result = parser.call(self, type)
+
+        return self.as_type(type) if result.nil?
+
+        unless result.is_a?(::KDL::Node)
+          raise ArgumentError, "expected parser to return an instance of ::KDL::Node, got `#{result.class}'"
+        end
+
+        result
+      end
+    end
+
+    private
+
+    def id_to_s(id)
+      StringDumper.stringify_identifier(id)
+    end
   end
 end

data/lib/kdl/string_dumper.rb

@@ -0,0 +1,45 @@
+module KDL
+  module StringDumper
+    class << self
+      def call(string)
+        %("#{string.each_char.map { |char| escape(char) }.join}")
+      end
+
+      def stringify_identifier(ident)
+        if bare_identifier?(ident)
+          ident
+        else
+          call(ident)
+        end
+      end
+
+      private
+
+      def print?(char)
+        ' ' <= char && char <= '\x7e'
+      end
+
+      def escape(char)
+        case char
+        when "\n" then '\n'
+        when "\r" then '\r'
+        when "\t" then '\t'
+        when '\\' then '\\\\'
+        when '"' then '\"'
+        when "\b" then '\b'
+        when "\f" then '\f'
+        else char
+        end
+      end
+
+      def unicode_escape(char)
+        "\\u{#{char.codepoints.first.to_s(16)}}"
+      end
+
+      def bare_identifier?(name)
+        escape_chars = '\\\/(){}<>;\[\]=,"'
+        name =~ /^([^0-9\-+\s#{escape_chars}][^\s#{escape_chars}]*|[\-+](?!true|false|null)[^0-9\s#{escape_chars}][^\s#{escape_chars}]*)$/
+      end
+    end
+  end
+end

data/lib/kdl/tokenizer.rb

@@ -1,12 +1,41 @@
+require 'bigdecimal'
+
 module KDL
   class Tokenizer
-    class Error < StandardError; end
+    class Error < StandardError
+      def initialize(message, line, column)
+        super("#{message} (#{line}:#{column})")
+      end
+    end
+
+    class Token
+      attr_reader :type, :value, :line, :column, :meta
+
+      def initialize(type, value, line, column, meta = {})
+        @type = type
+        @value = value
+        @line = line
+        @column = column
+        @meta = meta
+      end
+
+      def ==(other)
+        return false unless other.is_a?(Token)
+
+        type == other.type && value == other.value && line == other.line && column == other.column
+      end
+
+      def to_s
+        "#{value.inspect} (#{line}:#{column})"
+      end
+      alias inspect to_s
+    end
 
     attr_reader :index
 
     SYMBOLS = {
-      '{' => :LPAREN,
-      '}' => :RPAREN,
+      '{' => :LBRACE,
+      '}' => :RBRACE,
       '=' => :EQUALS,
       '＝' => :EQUALS,
       ';' => :SEMICOLON
@@ -20,10 +49,13 @@ module KDL
 
     NEWLINES = ["\u000A", "\u0085", "\u000C", "\u2028", "\u2029"]
 
-    NON_IDENTIFIER_CHARS = Regexp.escape "#{SYMBOLS.keys.join('')}\\<>[]\","
+    NON_IDENTIFIER_CHARS = Regexp.escape "#{SYMBOLS.keys.join('')}()/\\<>[]\","
     IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}\x0-\x20]/
     INITIAL_IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}0-9\x0-\x20]/
 
+    ALLOWED_IN_TYPE = [:ident, :string, :rawstring]
+    NOT_ALLOWED_AFTER_TYPE = [:single_line_comment, :multi_line_comment]
+
     def initialize(str, start = 0)
       @str = str
       @context = nil
@@ -32,11 +64,17 @@ module KDL
       @buffer = ""
       @done = false
       @previous_context = nil
+      @line = 1
+      @column = 1
+      @type_context = false
+      @last_token = nil
     end
 
     def next_token
       @context = nil
       @previous_context = nil
+      @line_at_start = @line
+      @column_at_start = @column
       loop do
         c = @str[@index]
         case @context
@@ -45,11 +83,11 @@ module KDL
           when '"'
             self.context = :string
             @buffer = ''
-            @index += 1
+            traverse(1)
           when 'r'
             if @str[@index + 1] == '"'
               self.context = :rawstring
-              @index += 2
+              traverse(2)
               @rawstring_hashes = 0
               @buffer = ''
               next
@@ -69,20 +107,20 @@ module KDL
             end
             self.context = :ident
             @buffer = c
-            @index += 1
+            traverse(1)
           when /[0-9\-+]/
             n = @str[@index + 1]
             if c == '0' && n =~ /[box]/
-              @index += 2
+              traverse(2)
               @buffer = ''
               self.context = case n
-                         when 'b' then :binary
-                         when 'o' then :octal
-                         when 'x' then :hexadecimal
-                         end
+                             when 'b' then :binary
+                             when 'o' then :octal
+                             when 'x' then :hexadecimal
+                             end
             else
               self.context = :decimal
-              @index += 1
+              traverse(1)
               @buffer = c
             end
           when '\\'
@@ -90,68 +128,80 @@ module KDL
             la = t.next_token[0]
             if la == :NEWLINE
               @index = t.index
+              new_line
             elsif la == :WS && (lan = t.next_token[0]) == :NEWLINE
               @index = t.index
+              new_line
             else
-              raise Error, "Unexpected '\\'"
+              raise_error "Unexpected '\\'"
             end
           when *SYMBOLS.keys
-            @index += 1
-            return [SYMBOLS[c], c]
+            return token(SYMBOLS[c], c).tap { traverse(1) }
           when "\r"
             n = @str[@index + 1]
             if n == "\n"
-              @index += 2
-              return [:NEWLINE, "#{c}#{n}"]
+              return token(:NEWLINE, "#{c}#{n}").tap do
+                traverse(2)
+                new_line
+              end
             else
-              @index += 1
-              return [:NEWLINE, c]
+              return token(:NEWLINE, c).tap do
+                traverse(1)
+                new_line
+              end
             end
           when *NEWLINES
-            @index += 1
-            return [:NEWLINE, c]
+            return token(:NEWLINE, c).tap do
+              traverse(1)
+              new_line
+            end
           when "/"
             if @str[@index + 1] == '/'
               self.context = :single_line_comment
-              @index += 2
+              traverse(2)
             elsif @str[@index + 1] == '*'
               self.context = :multi_line_comment
               @comment_nesting = 1
-              @index += 2
+              traverse(2)
             elsif @str[@index + 1] == '-'
-              @index += 2
-              return [:SLASHDASH, '/-']
+              return token(:SLASHDASH, '/-').tap { traverse(2) }
             else
               self.context = :ident
               @buffer = c
-              @index += 1
+              traverse(1)
             end
           when *WHITEPACE
             self.context = :whitespace
             @buffer = c
-            @index += 1
+            traverse(1)
           when nil
-            return [false, false] if @done
+            return [false, token(:EOF, :EOF)[1]] if @done
             @done = true
-            return [:EOF, '']
+            return token(:EOF, :EOF)
           when INITIAL_IDENTIFIER_CHARS
             self.context = :ident
             @buffer = c
-            @index += 1
+            traverse(1)
+          when '('
+            @type_context = true
+            return token(:LPAREN, c).tap { traverse(1) }
+          when ')'
+            @type_context = false
+            return token(:RPAREN, c).tap { traverse(1) }
           else
-            raise Error, "Unexpected character #{c.inspect}"
+            raise_error "Unexpected character #{c.inspect}"
           end
         when :ident
           case c
           when IDENTIFIER_CHARS
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             case @buffer
-            when 'true'  then return [:TRUE, true]
-            when 'false' then return [:FALSE, false]
-            when 'null'  then return [:NULL, nil]
-            else return [:IDENT, @buffer]
+            when 'true'  then return token(:TRUE, true)
+            when 'false' then return token(:FALSE, false)
+            when 'null'  then return token(:NULL, nil)
+            else return token(:IDENT, @buffer)
             end
           end
         when :string
@@ -159,18 +209,17 @@ module KDL
           when '\\'
             @buffer += c
             @buffer += @str[@index + 1]
-            @index += 2
+            traverse(2)
           when '"'
-            @index += 1
-            return [:STRING, convert_escapes(@buffer)]
+            return token(:STRING, convert_escapes(@buffer)).tap { traverse(1) }
           when nil
-            raise Error, "Unterminated string literal"
+            raise_error "Unterminated string literal"
           else
             @buffer += c
-            @index += 1
+            traverse(1)
           end
         when :rawstring
-          raise Error, "Unterminated rawstring literal" if c.nil?
+          raise_error "Unterminated rawstring literal" if c.nil?
 
           if c == '"'
             h = 0
@@ -178,17 +227,16 @@ module KDL
               h += 1
             end
             if h == @rawstring_hashes
-              @index += 1 + h
-              return [:RAWSTRING, @buffer]
+              return token(:RAWSTRING, @buffer).tap { traverse(1 + h) }
             end
           end
 
           @buffer += c
-          @index += 1
+          traverse(1)
         when :decimal
           case c
           when /[0-9.\-+_eE]/
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             return parse_decimal(@buffer)
@@ -196,7 +244,7 @@ module KDL
         when :hexadecimal
           case c
           when /[0-9a-fA-F_]/
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             return parse_hexadecimal(@buffer)
@@ -204,7 +252,7 @@ module KDL
         when :octal
           case c
           when /[0-7_]/
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             return parse_octal(@buffer)
@@ -212,7 +260,7 @@ module KDL
         when :binary
           case c
           when /[01_]/
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             return parse_binary(@buffer)
@@ -220,52 +268,80 @@ module KDL
         when :single_line_comment
           if NEWLINES.include?(c) || c == "\r"
             self.context = nil
+            @column_at_start = @column
             next
           elsif c.nil?
             @done = true
-            return [:EOF, '']
+            return token(:EOF, :EOF)
           else
-            @index += 1
+            traverse(1)
           end
         when :multi_line_comment
           if c == '/' && @str[@index + 1] == '*'
             @comment_nesting += 1
-            @index += 2
+            traverse(2)
           elsif c == '*' && @str[@index + 1] == '/'
             @comment_nesting -= 1
-            @index += 2
+            traverse(2)
             if @comment_nesting == 0
               revert_context
             end
           else
-            @index += 1
+            traverse(1)
           end
         when :whitespace
           if WHITEPACE.include?(c)
-            @index += 1
+            traverse(1)
             @buffer += c
           elsif c == "\\"
             t = Tokenizer.new(@str, @index + 1)
             la = t.next_token[0]
             if la == :NEWLINE
               @index = t.index
+              new_line
             elsif (la == :WS && (lan = t.next_token[0]) == :NEWLINE)
               @index = t.index
+              new_line
             else
-              raise Error, "Unexpected '\\'"
+              raise_error "Unexpected '\\'"
             end
           elsif c == "/" && @str[@index + 1] == '*'
             self.context = :multi_line_comment
             @comment_nesting = 1
-            @index += 2
+            traverse(2)
           else
-            return [:WS, @buffer]
+            return token(:WS, @buffer)
           end
         end
       end
     end
 
+    private
+
+    def token(type, value, **meta)
+      @last_token = [type, Token.new(type, value, @line_at_start, @column_at_start, meta)]
+    end
+
+    def traverse(n = 1)
+      @column += n
+      @index += n
+    end
+
+    def raise_error(message)
+      raise Error.new(message, @line, @column)
+    end
+
+    def new_line
+      @column = 1
+      @line += 1
+    end
+
     def context=(val)
+      if @type_context && !ALLOWED_IN_TYPE.include?(val)
+        raise_error "#{val} context not allowed in type declaration"
+      elsif @last_token && @last_token[0] == :RPAREN && NOT_ALLOWED_AFTER_TYPE.include?(val)
+        raise_error 'Comments are not allowed after a type declaration'
+      end
       @previous_context = @context
       @context = val
     end
@@ -275,23 +351,38 @@ module KDL
       @previous_context = nil
     end
 
-    private
-
     def parse_decimal(s)
-      return [:FLOAT, Float(munch_underscores(s))] if s =~ /[.eE]/
-      [:INTEGER, Integer(munch_underscores(s), 10)]
+      return parse_float(s) if s =~ /[.E]/i
+
+      token(:INTEGER, Integer(munch_underscores(s), 10), format: '%d')
     end
-    
+
+    def parse_float(s)
+      match, _, fraction, exponent = *s.match(/^([-+]?[\d_]+)(?:\.(\d+))?(?:[eE]([-+]?[\d_]+))?$/)
+      raise_error "Invalid floating point value #{s}" if match.nil?
+
+      s = munch_underscores(s)
+
+      decimals = fraction.nil? ? 0 : fraction.size
+      value = Float(s)
+      scientific = value.abs >= 100 || (exponent && exponent.to_i.abs >= 2)
+      if value.infinite? || (value.zero? && exponent.to_i < 0)
+        token(:FLOAT, BigDecimal(s))
+      else
+        token(:FLOAT, value, format: scientific ? "%.#{decimals}E" : nil)
+      end
+    end
+
     def parse_hexadecimal(s)
-      [:INTEGER, Integer(munch_underscores(s), 16)]
+      token(:INTEGER, Integer(munch_underscores(s), 16))
     end
-    
+
     def parse_octal(s)
-      [:INTEGER, Integer(munch_underscores(s), 8)]
+      token(:INTEGER, Integer(munch_underscores(s), 8))
     end
-    
+
     def parse_binary(s)
-      [:INTEGER, Integer(munch_underscores(s), 2)]
+      token(:INTEGER, Integer(munch_underscores(s), 2))
     end
 
     def munch_underscores(s)
@@ -308,12 +399,13 @@ module KDL
         when '\"' then "\""
         when '\b' then "\b"
         when '\f' then "\f"
-        else raise Error, "Unexpected escape #{m.inspect}"
+        when '\/' then "/"
+        else raise_error "Unexpected escape #{m.inspect}"
         end
       end.gsub(/\\u\{[0-9a-fA-F]{0,6}\}/) do |m|
         i = Integer(m[3..-2], 16)
         if i < 0 || i > 0x10FFFF
-          raise Error, "Invalid code point #{u}"
+          raise_error "Invalid code point #{u}"
         end
         i.chr(Encoding::UTF_8)
       end

data/lib/kdl/types/base64.rb

@@ -0,0 +1,15 @@
+require 'base64'
+
+module KDL
+  module Types
+    class Base64 < Value
+      def self.call(value, type = 'base64')
+        return nil unless value.is_a? ::KDL::Value::String
+
+        data = ::Base64.decode64(value.value)
+        new(data, type: type)
+      end
+    end
+    MAPPING['base64'] = Base64
+  end
+end