kdl 1.0.6 → 2.0.1

data/lib/kdl/v1/string_dumper.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module KDL
+  module V1
+    module StringDumper
+      include ::KDL::StringDumper
+
+      def call(string)
+        %("#{string.each_char.map { |char| escape(char) }.join}")
+      end
+
+      def stringify_identifier(ident)
+        if bare_identifier?(ident)
+          ident
+        else
+          call(ident)
+        end
+      end
+
+      private
+
+      def bare_identifier?(name)
+        escape_chars = '\\\/(){}<>;\[\]=,"'
+        name =~ /^([^0-9\-+\s#{escape_chars}][^\s#{escape_chars}]*|[\-+](?!true|false|null)[^0-9\s#{escape_chars}][^\s#{escape_chars}]*)$/
+      end
+
+      extend self
+    end
+  end
+end

data/lib/kdl/v1/tokenizer.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+
+module KDL
+  module V1
+    class Tokenizer < KDL::Tokenizer
+      NON_IDENTIFIER_CHARS = Regexp.escape "#{SYMBOLS.keys.join}()/\\<>[]\",#{WHITESPACE.join}#{OTHER_NON_IDENTIFIER_CHARS.join}"
+      IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}]/
+      INITIAL_IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}0-9]/
+
+      def next_token
+        @context = nil
+        @previous_context = nil
+        @line_at_start = @line
+        @column_at_start = @column
+        loop do
+          c = self[@index]
+          case @context
+          when nil
+            case c
+            when '"'
+              self.context = :string
+              @buffer = +''
+              traverse(1)
+            when 'r'
+              if @str[@index + 1] == '"'
+                self.context = :rawstring
+                traverse(2)
+                @rawstring_hashes = 0
+                @buffer = +''
+                next
+              elsif @str[@index + 1] == '#'
+                i = @index + 1
+                @rawstring_hashes = 0
+                while @str[i] == '#'
+                  @rawstring_hashes += 1
+                  i += 1
+                end
+                if @str[i] == '"'
+                  self.context = :rawstring
+                  @index = i + 1
+                  @buffer = +''
+                  next
+                end
+              end
+              self.context = :ident
+              @buffer = +c
+              traverse(1)
+            when '-'
+              n = self[@index + 1]
+              if n =~ /[0-9]/
+                n2 = self[@index + 2]
+                if n == '0' && n2 =~ /[box]/
+                  self.context = integer_context(n2)
+                  traverse(3)
+                else
+                  self.context = :decimal
+                  traverse(1)
+                end
+              else
+                self.context = :ident
+                traverse(1)
+              end
+              @buffer = +c
+            when /[0-9+]/
+              n = self[@index + 1]
+              if c == '0' && n =~ /[box]/
+                traverse(2)
+                @buffer = +''
+                self.context = integer_context(n)
+              else
+                self.context = :decimal
+                @buffer = +c
+                traverse(1)
+              end
+            when '\\'
+              t = Tokenizer.new(@str, @index + 1)
+              la = t.next_token
+              if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF)
+                traverse_to(t.index)
+                @buffer = "#{c}#{la[1].value}"
+                @buffer << "\n" if lan == :NEWLINE
+                self.context = :whitespace
+              else
+                raise_error "Unexpected '\\' (#{la[0]})"
+              end
+            when *SYMBOLS.keys
+              return token(SYMBOLS[c], -c).tap { traverse(1) }
+            when *NEWLINES, "\r"
+              nl = expect_newline
+              return token(:NEWLINE, -nl).tap do
+                traverse(nl.length)
+              end
+            when "/"
+              if self[@index + 1] == '/'
+                self.context = :single_line_comment
+                traverse(2)
+              elsif self[@index + 1] == '*'
+                self.context = :multi_line_comment
+                @comment_nesting = 1
+                traverse(2)
+              elsif self[@index + 1] == '-'
+                return token(:SLASHDASH, '/-').tap { traverse(2) }
+              else
+                self.context = :ident
+                @buffer = +c
+                traverse(1)
+              end
+            when *WHITESPACE
+              self.context = :whitespace
+              @buffer = +c
+              traverse(1)
+            when nil
+              return [false, token(:EOF, :EOF)[1]] if @done
+
+              @done = true
+              return token(:EOF, :EOF)
+            when INITIAL_IDENTIFIER_CHARS
+              self.context = :ident
+              @buffer = +c
+              traverse(1)
+            when '('
+              @type_context = true
+              return token(:LPAREN, -c).tap { traverse(1) }
+            when ')'
+              @type_context = false
+              return token(:RPAREN, -c).tap { traverse(1) }
+            else
+              raise_error "Unexpected character #{c.inspect}"
+            end
+          when :ident
+            case c
+            when IDENTIFIER_CHARS
+              traverse(1)
+              @buffer << c
+            else
+              case @buffer
+              when 'true'  then return token(:TRUE, true)
+              when 'false' then return token(:FALSE, false)
+              when 'null'  then return token(:NULL, nil)
+              else return token(:IDENT, -@buffer)
+              end
+            end
+          when :string
+            case c
+            when '\\'
+              c2 = self[@index + 1]
+              if c2.match?(NEWLINES_PATTERN)
+                i = 2
+                while self[@index + i].match?(NEWLINES_PATTERN)
+                  i+=1
+                end
+                traverse(i)
+              else
+                @buffer << c
+                @buffer << c2
+                traverse(2)
+              end
+            when '"'
+              return token(:STRING, -unescape(@buffer)).tap { traverse(1) }
+            when nil
+              raise_error "Unterminated string literal"
+            else
+              @buffer << c
+              traverse(1)
+            end
+          when :rawstring
+            raise_error "Unterminated rawstring literal" if c.nil?
+
+            if c == '"'
+              h = 0
+              h += 1 while self[@index + 1 + h] == '#' && h < @rawstring_hashes
+              if h == @rawstring_hashes
+                return token(:RAWSTRING, -@buffer).tap { traverse(1 + h) }
+              end
+            end
+
+            @buffer << c
+            traverse(1)
+          when :decimal
+            case c
+            when /[0-9.\-+_eE]/
+              traverse(1)
+              @buffer << c
+            else
+              return parse_decimal(@buffer)
+            end
+          when :hexadecimal
+            case c
+            when /[0-9a-fA-F_]/
+              traverse(1)
+              @buffer << c
+            else
+              return parse_hexadecimal(@buffer)
+            end
+          when :octal
+            case c
+            when /[0-7_]/
+              traverse(1)
+              @buffer << c
+            else
+              return parse_octal(@buffer)
+            end
+          when :binary
+            case c
+            when /[01_]/
+              traverse(1)
+              @buffer << c
+            else
+              return parse_binary(@buffer)
+            end
+          when :single_line_comment
+            case c
+            when *NEWLINES, "\r"
+              self.context = nil
+              @column_at_start = @column
+              next
+            when nil
+              @done = true
+              return token(:EOF, :EOF)
+            else
+              traverse(1)
+            end
+          when :multi_line_comment
+            if c == '/' && self[@index + 1] == '*'
+              @comment_nesting += 1
+              traverse(2)
+            elsif c == '*' && self[@index + 1] == '/'
+              @comment_nesting -= 1
+              traverse(2)
+              if @comment_nesting == 0
+                revert_context
+              end
+            else
+              traverse(1)
+            end
+          when :whitespace
+            if WHITESPACE.include?(c)
+              traverse(1)
+              @buffer << c
+            elsif c == "/" && self[@index + 1] == '*'
+              self.context = :multi_line_comment
+              @comment_nesting = 1
+              traverse(2)
+            elsif c == "\\"
+              t = Tokenizer.new(@str, @index + 1)
+              la = t.next_token
+              if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF)
+                traverse_to(t.index)
+                @buffer << "#{c}#{la[1].value}"
+                @buffer << "\n" if lan == :NEWLINE
+              else
+                raise_error "Unexpected '\\' (#{la[0]})"
+              end
+            else
+              return token(:WS, -@buffer)
+            end
+          else
+            # :nocov:
+            raise_error "Unknown context `#{@context}'"
+            # :nocov:
+          end
+        end
+      end
+
+      private
+
+      def allowed_in_type?(val)
+        %i[ident string rawstring].include?(val)
+      end
+
+      def allowed_after_type?(val)
+        !%i[single_line_comment multi_line_comment].include?(val)
+      end
+
+      def unescape(string)
+        string.gsub(/\\[^u]/) do |m|
+          case m
+          when '\n' then "\n"
+          when '\r' then "\r"
+          when '\t' then "\t"
+          when '\\\\' then "\\"
+          when '\"' then "\""
+          when '\b' then "\b"
+          when '\f' then "\f"
+          when '\/' then "/"
+          else raise_error "Unexpected escape #{m.inspect}"
+          end
+        end.gsub(/\\u\{[0-9a-fA-F]{0,6}\}/) do |m|
+          i = Integer(m[3..-2], 16)
+          if i < 0 || i > 0x10FFFF
+            raise_error "Invalid code point #{u}"
+          end
+          i.chr(Encoding::UTF_8)
+        end
+      end
+    end
+  end
+end

data/lib/kdl/v1/value.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module KDL
+  module V1
+    class Value < ::KDL::Value
+      module Methods
+        def to_s
+          return stringify_value unless type
+
+          "(#{StringDumper.stringify_identifier type})#{stringify_value}"
+        end
+
+        def ==(other)
+          return self == other.value if other.is_a?(self.class.superclass)
+
+          value == other
+        end
+
+        def version
+          1
+        end
+
+        def to_v1
+          self
+        end
+
+        def to_v2
+          self.class.superclass.new(value, format:, type:)
+        end
+      end
+
+      include Methods
+
+      class Int < ::KDL::Value::Int
+        include Methods
+      end
+
+      class Float < ::KDL::Value::Float
+        include Methods
+
+        def stringify_value
+          if value.nan? || value.infinite?
+            warn "[WARNING] Attempting to serialize non-finite Float using KDL v1"
+            return Null.stringify_value
+          end
+          super
+        end
+      end
+
+      class Boolean < ::KDL::Value::Boolean
+        include Methods
+
+        def stringify_value
+          value.to_s
+        end
+      end
+
+      class String < ::KDL::Value::String
+        include Methods
+
+        def stringify_value
+          StringDumper.call(value)
+        end
+      end
+
+      class NullImpl < ::KDL::Value::NullImpl
+        include Methods
+
+        def stringify_value
+          "null"
+        end
+
+        def to_v2
+          type ? ::KDL::Value::NullImpl.new(type:) : ::KDL::Value::Null
+        end
+      end
+      Null = NullImpl.new
+
+      def self.from(value)
+        case value
+        when ::String then String.new(value)
+        when Integer then Int.new(value)
+        when ::Float then Float.new(value)
+        when TrueClass, FalseClass then Boolean.new(value)
+        when NilClass then Null
+        else raise Error("Unsupported value type: #{value.class}")
+        end
+      end
+    end
+  end
+end

data/lib/kdl/v1.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "kdl/v1/tokenizer"
+require "kdl/v1/document"
+require "kdl/v1/value"
+require "kdl/v1/node"
+require "kdl/v1/string_dumper"
+require "kdl/v1/kdl.tab"
+
+module KDL
+  module V1
+  end
+end

data/lib/kdl/value.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module KDL
   class Value
     attr_reader :value, :format, :type
@@ -15,18 +17,30 @@ module KDL
         result = parser.call(self, type)
         return self.as_type(type) if result.nil?
 
-        unless result.is_a?(::KDL::Value)
-          raise ArgumentError, "expected parser to return an instance of ::KDL::Value, got `#{result.class}'"
+        unless result.is_a?(::KDL::Value::Custom)
+          raise ArgumentError, "expected parser to return an instance of ::KDL::Value::Custom, got `#{result.class}'"
         end
 
         result
       end
     end
 
+    def ==(other)
+      return self == other.value if other.is_a?(self.class)
+
+      value == other
+    end
+
     def to_s
       return stringify_value unless type
 
-      "(#{StringDumper.stringify_identifier type})#{stringify_value}"
+      "(#{StringDumper.call type})#{stringify_value}"
+    end
+
+    def inspect
+      return value.inspect unless type
+
+      "(#{type.inspect})#{value.inspect}"
     end
 
     def stringify_value
@@ -35,31 +49,65 @@ module KDL
       value.to_s
     end
 
+    def version
+      2
+    end
+
+    def to_v2
+      self
+    end
+
+    def method_missing(name, *args, **kwargs, &block)
+      value.public_send(name, *args, **kwargs, &block)
+    end
+
+    def respond_to_missing?(name, include_all = false)
+      value.respond_to?(name, include_all)
+    end
+
     class Int < Value
-      def ==(other)
-        other.is_a?(Int) && value == other.value
+      def to_v1
+        V1::Value::Int.new(value, format:, type:)
       end
     end
 
     class Float < Value
       def ==(other)
-        other.is_a?(Float) && value == other.value
+        return self == other.value if other.is_a?(Float)
+        return other.nan? if value.nan?
+
+        value == other
       end
 
       def stringify_value
+        return '#nan'  if value.nan?
+        return '#inf'  if value == ::Float::INFINITY
+        return '#-inf' if value == -::Float::INFINITY
         return super.upcase unless value.is_a?(BigDecimal)
 
         sign, digits, _, exponent = value.split
-        s = sign.negative? ? '-' : ''
-        s += "#{digits[0]}.#{digits[1..-1]}"
-        s += "E#{exponent.negative? ? '' : '+'}#{exponent - 1}"
+        s = +''
+        s << '-' if sign.negative?
+        s << "#{digits[0]}.#{digits[1..-1]}"
+        s << "E#{exponent.negative? ? '' : '+'}#{exponent - 1}"
         s
       end
+
+      def to_v1
+        if value.nan? || value.infinite?
+          warn "[WARNING] Converting non-finite Float to KDL v1"
+        end
+        V1::Value::Float.new(value, format:, type:)
+      end
     end
 
     class Boolean < Value
-      def ==(other)
-        other.is_a?(Boolean) && value == other.value
+      def stringify_value
+        "##{value}"
+      end
+
+      def to_v1
+        V1::Value::Boolean.new(value, format:, type:)
       end
     end
 
@@ -68,8 +116,8 @@ module KDL
         StringDumper.call(value)
       end
 
-      def ==(other)
-        other.is_a?(String) && value == other.value
+      def to_v1
+        V1::Value::String.new(value, format:, type:)
       end
     end
 
@@ -79,15 +127,39 @@ module KDL
       end
 
       def stringify_value
-        "null"
+        "#null"
       end
 
       def ==(other)
-        other.is_a?(NullImpl)
+        other.is_a?(NullImpl) || other.nil?
+      end
+
+      def to_v1
+        type ? V1::Value::NullImpl.new(type:) : V1::Value::Null
       end
     end
     Null = NullImpl.new
 
+    class Custom < Value
+      attr_reader :oriinal_value
+
+      def self.call(value, type)
+        new(value, type:)
+      end
+
+      def version
+        nil
+      end
+
+      def to_v1
+        self
+      end
+
+      def to_v2
+        self
+      end
+    end
+
     def self.from(value)
       case value
       when ::String then String.new(value)

data/lib/kdl/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module KDL
-  VERSION = "1.0.6"
+  VERSION = "2.0.1"
 end

data/lib/kdl.rb

@@ -1,14 +1,60 @@
+# frozen_string_literal: true
+
 require "kdl/version"
+require "kdl/error"
 require "kdl/tokenizer"
 require "kdl/document"
 require "kdl/value"
 require "kdl/node"
 require "kdl/string_dumper"
 require "kdl/types"
+require "kdl/parser_common"
 require "kdl/kdl.tab"
+require "kdl/v1"
 
 module KDL
+  class << self
+    attr_accessor :default_version
+    attr_accessor :default_output_version
+  end
+
   def self.parse_document(input, options = {})
-    Parser.new.parse(input, options)
+    warn "[DEPRECATION] `KDL.parse_document' is deprecated. Please use `KDL.parse' instead."
+    parse(input, **options)
+  end
+
+  def self.parse(input, version: default_version, output_version: default_output_version, **options)
+    case version
+    when 2
+      Parser.new(output_module: output_module(output_version || 2), **options).parse(input)
+    when 1
+      V1::Parser.new.parse(input, output_module: output_module(output_version || 1), **options)
+    when nil
+      auto_parse(input, output_version:, **options)
+    else
+      raise UnsupportedVersionError.new("Unsupported version '#{version}'", version)
+    end
+  end
+
+  def self.load_file(filespec, **options)
+    parse(File.read(filespec, encoding: Encoding::UTF_8), **options)
+  end
+
+  def self.auto_parse(input, output_version: default_output_version, **options)
+    parse(input, version: 2, output_version: output_version || 2, **options)
+  rescue VersionMismatchError => e
+    parse(input, version: e.version, output_version: output_version || e.version, **options)
+  rescue Tokenizer::Error, Racc::ParseError => e
+    parse(input, version: 1, output_version: output_version || 1, **options) rescue raise e
+  end
+
+  def self.output_module(version)
+    case version
+    when 1 then KDL::V1
+    when 2 then KDL
+    else
+      warn "Unknown output_version `#{version}', defaulting to v2"
+      KDL
+    end
   end
 end