kdl 0.1.0 → 1.0.0.rc2

data/lib/kdl/tokenizer.rb

@@ -1,12 +1,41 @@
+require 'bigdecimal'
+
 module KDL
   class Tokenizer
-    class Error < StandardError; end
+    class Error < StandardError
+      def initialize(message, line, column)
+        super("#{message} (#{line}:#{column})")
+      end
+    end
+
+    class Token
+      attr_reader :type, :value, :line, :column, :meta
+
+      def initialize(type, value, line, column, meta = {})
+        @type = type
+        @value = value
+        @line = line
+        @column = column
+        @meta = meta
+      end
+
+      def ==(other)
+        return false unless other.is_a?(Token)
+
+        type == other.type && value == other.value && line == other.line && column == other.column
+      end
+
+      def to_s
+        "#{value.inspect} (#{line}:#{column})"
+      end
+      alias inspect to_s
+    end
 
     attr_reader :index
 
     SYMBOLS = {
-      '{' => :LPAREN,
-      '}' => :RPAREN,
+      '{' => :LBRACE,
+      '}' => :RBRACE,
       '=' => :EQUALS,
       '＝' => :EQUALS,
       ';' => :SEMICOLON
@@ -20,10 +49,13 @@ module KDL
 
     NEWLINES = ["\u000A", "\u0085", "\u000C", "\u2028", "\u2029"]
 
-    NON_IDENTIFIER_CHARS = Regexp.escape "#{SYMBOLS.keys.join('')}\\<>[]\","
+    NON_IDENTIFIER_CHARS = Regexp.escape "#{SYMBOLS.keys.join('')}()/\\<>[]\","
     IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}\x0-\x20]/
     INITIAL_IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}0-9\x0-\x20]/
 
+    ALLOWED_IN_TYPE = [:ident, :string, :rawstring]
+    NOT_ALLOWED_AFTER_TYPE = [:single_line_comment, :multi_line_comment]
+
     def initialize(str, start = 0)
       @str = str
       @context = nil
@@ -32,11 +64,17 @@ module KDL
       @buffer = ""
       @done = false
       @previous_context = nil
+      @line = 1
+      @column = 1
+      @type_context = false
+      @last_token = nil
     end
 
     def next_token
       @context = nil
       @previous_context = nil
+      @line_at_start = @line
+      @column_at_start = @column
       loop do
         c = @str[@index]
         case @context
@@ -45,11 +83,11 @@ module KDL
           when '"'
             self.context = :string
             @buffer = ''
-            @index += 1
+            traverse(1)
           when 'r'
             if @str[@index + 1] == '"'
               self.context = :rawstring
-              @index += 2
+              traverse(2)
               @rawstring_hashes = 0
               @buffer = ''
               next
@@ -69,20 +107,20 @@ module KDL
             end
             self.context = :ident
             @buffer = c
-            @index += 1
+            traverse(1)
           when /[0-9\-+]/
             n = @str[@index + 1]
             if c == '0' && n =~ /[box]/
-              @index += 2
+              traverse(2)
               @buffer = ''
               self.context = case n
-                         when 'b' then :binary
-                         when 'o' then :octal
-                         when 'x' then :hexadecimal
-                         end
+                             when 'b' then :binary
+                             when 'o' then :octal
+                             when 'x' then :hexadecimal
+                             end
             else
               self.context = :decimal
-              @index += 1
+              traverse(1)
               @buffer = c
             end
           when '\\'
@@ -90,68 +128,80 @@ module KDL
             la = t.next_token[0]
             if la == :NEWLINE
               @index = t.index
+              new_line
             elsif la == :WS && (lan = t.next_token[0]) == :NEWLINE
               @index = t.index
+              new_line
             else
-              raise Error, "Unexpected '\\'"
+              raise_error "Unexpected '\\'"
             end
           when *SYMBOLS.keys
-            @index += 1
-            return [SYMBOLS[c], c]
+            return token(SYMBOLS[c], c).tap { traverse(1) }
           when "\r"
             n = @str[@index + 1]
             if n == "\n"
-              @index += 2
-              return [:NEWLINE, "#{c}#{n}"]
+              return token(:NEWLINE, "#{c}#{n}").tap do
+                traverse(2)
+                new_line
+              end
             else
-              @index += 1
-              return [:NEWLINE, c]
+              return token(:NEWLINE, c).tap do
+                traverse(1)
+                new_line
+              end
             end
           when *NEWLINES
-            @index += 1
-            return [:NEWLINE, c]
+            return token(:NEWLINE, c).tap do
+              traverse(1)
+              new_line
+            end
           when "/"
             if @str[@index + 1] == '/'
               self.context = :single_line_comment
-              @index += 2
+              traverse(2)
             elsif @str[@index + 1] == '*'
               self.context = :multi_line_comment
               @comment_nesting = 1
-              @index += 2
+              traverse(2)
             elsif @str[@index + 1] == '-'
-              @index += 2
-              return [:SLASHDASH, '/-']
+              return token(:SLASHDASH, '/-').tap { traverse(2) }
             else
               self.context = :ident
               @buffer = c
-              @index += 1
+              traverse(1)
             end
           when *WHITEPACE
             self.context = :whitespace
             @buffer = c
-            @index += 1
+            traverse(1)
           when nil
-            return [false, false] if @done
+            return [false, token(:EOF, :EOF)[1]] if @done
             @done = true
-            return [:EOF, '']
+            return token(:EOF, :EOF)
           when INITIAL_IDENTIFIER_CHARS
             self.context = :ident
             @buffer = c
-            @index += 1
+            traverse(1)
+          when '('
+            @type_context = true
+            return token(:LPAREN, c).tap { traverse(1) }
+          when ')'
+            @type_context = false
+            return token(:RPAREN, c).tap { traverse(1) }
           else
-            raise Error, "Unexpected character #{c.inspect}"
+            raise_error "Unexpected character #{c.inspect}"
           end
         when :ident
           case c
           when IDENTIFIER_CHARS
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             case @buffer
-            when 'true'  then return [:TRUE, true]
-            when 'false' then return [:FALSE, false]
-            when 'null'  then return [:NULL, nil]
-            else return [:IDENT, @buffer]
+            when 'true'  then return token(:TRUE, true)
+            when 'false' then return token(:FALSE, false)
+            when 'null'  then return token(:NULL, nil)
+            else return token(:IDENT, @buffer)
             end
           end
         when :string
@@ -159,18 +209,17 @@ module KDL
           when '\\'
             @buffer += c
             @buffer += @str[@index + 1]
-            @index += 2
+            traverse(2)
           when '"'
-            @index += 1
-            return [:STRING, convert_escapes(@buffer)]
+            return token(:STRING, convert_escapes(@buffer)).tap { traverse(1) }
           when nil
-            raise Error, "Unterminated string literal"
+            raise_error "Unterminated string literal"
           else
             @buffer += c
-            @index += 1
+            traverse(1)
           end
         when :rawstring
-          raise Error, "Unterminated rawstring literal" if c.nil?
+          raise_error "Unterminated rawstring literal" if c.nil?
 
           if c == '"'
             h = 0
@@ -178,17 +227,16 @@ module KDL
               h += 1
             end
             if h == @rawstring_hashes
-              @index += 1 + h
-              return [:RAWSTRING, @buffer]
+              return token(:RAWSTRING, @buffer).tap { traverse(1 + h) }
             end
           end
 
           @buffer += c
-          @index += 1
+          traverse(1)
         when :decimal
           case c
           when /[0-9.\-+_eE]/
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             return parse_decimal(@buffer)
@@ -196,7 +244,7 @@ module KDL
         when :hexadecimal
           case c
           when /[0-9a-fA-F_]/
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             return parse_hexadecimal(@buffer)
@@ -204,7 +252,7 @@ module KDL
         when :octal
           case c
           when /[0-7_]/
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             return parse_octal(@buffer)
@@ -212,7 +260,7 @@ module KDL
         when :binary
           case c
           when /[01_]/
-            @index += 1
+            traverse(1)
             @buffer += c
           else
             return parse_binary(@buffer)
@@ -220,52 +268,80 @@ module KDL
         when :single_line_comment
           if NEWLINES.include?(c) || c == "\r"
             self.context = nil
+            @column_at_start = @column
             next
           elsif c.nil?
             @done = true
-            return [:EOF, '']
+            return token(:EOF, :EOF)
           else
-            @index += 1
+            traverse(1)
           end
         when :multi_line_comment
           if c == '/' && @str[@index + 1] == '*'
             @comment_nesting += 1
-            @index += 2
+            traverse(2)
           elsif c == '*' && @str[@index + 1] == '/'
             @comment_nesting -= 1
-            @index += 2
+            traverse(2)
             if @comment_nesting == 0
               revert_context
             end
           else
-            @index += 1
+            traverse(1)
           end
         when :whitespace
           if WHITEPACE.include?(c)
-            @index += 1
+            traverse(1)
             @buffer += c
           elsif c == "\\"
             t = Tokenizer.new(@str, @index + 1)
             la = t.next_token[0]
             if la == :NEWLINE
               @index = t.index
+              new_line
             elsif (la == :WS && (lan = t.next_token[0]) == :NEWLINE)
               @index = t.index
+              new_line
             else
-              raise Error, "Unexpected '\\'"
+              raise_error "Unexpected '\\'"
             end
           elsif c == "/" && @str[@index + 1] == '*'
             self.context = :multi_line_comment
             @comment_nesting = 1
-            @index += 2
+            traverse(2)
           else
-            return [:WS, @buffer]
+            return token(:WS, @buffer)
           end
         end
       end
     end
 
+    private
+
+    def token(type, value, **meta)
+      @last_token = [type, Token.new(type, value, @line_at_start, @column_at_start, meta)]
+    end
+
+    def traverse(n = 1)
+      @column += n
+      @index += n
+    end
+
+    def raise_error(message)
+      raise Error.new(message, @line, @column)
+    end
+
+    def new_line
+      @column = 1
+      @line += 1
+    end
+
     def context=(val)
+      if @type_context && !ALLOWED_IN_TYPE.include?(val)
+        raise_error "#{val} context not allowed in type declaration"
+      elsif @last_token && @last_token[0] == :RPAREN && NOT_ALLOWED_AFTER_TYPE.include?(val)
+        raise_error 'Comments are not allowed after a type declaration'
+      end
       @previous_context = @context
       @context = val
     end
@@ -275,23 +351,38 @@ module KDL
       @previous_context = nil
     end
 
-    private
-
     def parse_decimal(s)
-      return [:FLOAT, Float(munch_underscores(s))] if s =~ /[.eE]/
-      [:INTEGER, Integer(munch_underscores(s), 10)]
+      return parse_float(s) if s =~ /[.E]/i
+
+      token(:INTEGER, Integer(munch_underscores(s), 10), format: '%d')
     end
-    
+
+    def parse_float(s)
+      match, _, fraction, exponent = *s.match(/^([-+]?[\d_]+)(?:\.(\d+))?(?:[eE]([-+]?[\d_]+))?$/)
+      raise_error "Invalid floating point value #{s}" if match.nil?
+
+      s = munch_underscores(s)
+
+      decimals = fraction.nil? ? 0 : fraction.size
+      value = Float(s)
+      scientific = value.abs >= 100 || (exponent && exponent.to_i.abs >= 2)
+      if value.infinite? || (value.zero? && exponent.to_i < 0)
+        token(:FLOAT, BigDecimal(s))
+      else
+        token(:FLOAT, value, format: scientific ? "%.#{decimals}E" : nil)
+      end
+    end
+
     def parse_hexadecimal(s)
-      [:INTEGER, Integer(munch_underscores(s), 16)]
+      token(:INTEGER, Integer(munch_underscores(s), 16))
     end
-    
+
     def parse_octal(s)
-      [:INTEGER, Integer(munch_underscores(s), 8)]
+      token(:INTEGER, Integer(munch_underscores(s), 8))
     end
-    
+
     def parse_binary(s)
-      [:INTEGER, Integer(munch_underscores(s), 2)]
+      token(:INTEGER, Integer(munch_underscores(s), 2))
     end
 
     def munch_underscores(s)
@@ -308,12 +399,13 @@ module KDL
         when '\"' then "\""
         when '\b' then "\b"
         when '\f' then "\f"
-        else raise Error, "Unexpected escape #{m.inspect}"
+        when '\/' then "/"
+        else raise_error "Unexpected escape #{m.inspect}"
         end
       end.gsub(/\\u\{[0-9a-fA-F]{0,6}\}/) do |m|
         i = Integer(m[3..-2], 16)
         if i < 0 || i > 0x10FFFF
-          raise Error, "Invalid code point #{u}"
+          raise_error "Invalid code point #{u}"
         end
         i.chr(Encoding::UTF_8)
       end

data/lib/kdl/types/base64.rb

@@ -0,0 +1,15 @@
+require 'base64'
+
+module KDL
+  module Types
+    class Base64 < Value
+      def self.call(value, type = 'base64')
+        return nil unless value.is_a? ::KDL::Value::String
+
+        data = ::Base64.decode64(value.value)
+        new(data, type: type)
+      end
+    end
+    MAPPING['base64'] = Base64
+  end
+end