hocon 0.0.7 → 0.1.0

data/lib/hocon/impl/substitution_expression.rb

@@ -0,0 +1,38 @@
+require 'hocon/impl'
+
+
+class Hocon::Impl::SubstitutionExpression
+
+  def initialize(path, optional)
+    @path = path
+    @optional = optional
+  end
+  attr_reader :path, :optional
+
+  def change_path(new_path)
+    if new_path == @path
+      self
+    else
+      Hocon::Impl::SubstitutionExpression.new(new_path, @optional)
+    end
+  end
+
+  def to_s
+    "${#{@optional ? "?" : ""}#{@path.render}}"
+  end
+
+  def ==(other)
+    if other.is_a? Hocon::Impl::SubstitutionExpression
+      other.path == @path && other.optional == @optional
+    else
+      false
+    end
+  end
+
+  def hash
+    h = 41 * (41 + @path.hash)
+    h = 41 * (h + (optional ? 1 : 0))
+
+    h
+  end
+end

data/lib/hocon/impl/token.rb

@@ -1,14 +1,18 @@
+# encoding: utf-8
+
 require 'hocon/impl'
 require 'hocon/impl/token_type'
 
 class Hocon::Impl::Token
-  def self.new_without_origin(token_type, debug_string)
-    Hocon::Impl::Token.new(token_type, nil, debug_string)
+  attr_reader :token_type, :token_text
+  def self.new_without_origin(token_type, debug_string, token_text)
+    Hocon::Impl::Token.new(token_type, nil, token_text, debug_string)
   end
 
-  def initialize(token_type, origin, debug_string = nil)
+  def initialize(token_type, origin, token_text = nil, debug_string = nil)
     @token_type = token_type
     @origin = origin
+    @token_text = token_text
     @debug_string = debug_string
   end
 
@@ -29,4 +33,13 @@ class Hocon::Impl::Token
       Hocon::Impl::TokenType.name(@token_type)
     end
   end
-end
+
+  def ==(other)
+    # @origin deliberately left out
+    other.is_a?(Hocon::Impl::Token) && @token_type == other.token_type
+  end
+
+  def hash
+    @token_type.hash
+  end
+end

data/lib/hocon/impl/token_type.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'hocon/impl'
 
 class Hocon::Impl::TokenType
@@ -17,6 +19,7 @@ class Hocon::Impl::TokenType
   PROBLEM = 13
   COMMENT = 14
   PLUS_EQUALS = 15
+  IGNORED_WHITESPACE = 16
 
   def self.name(token_type)
     case token_type
@@ -36,7 +39,8 @@ class Hocon::Impl::TokenType
       when PROBLEM then "PROBLEM"
       when COMMENT then "COMMENT"
       when PLUS_EQUALS then "PLUS_EQUALS"
-      else raise ConfigBugError, "Unrecognized token type #{token_type}"
+      when IGNORED_WHITESPACE then "IGNORED_WHITESPACE"
+      else raise ConfigBugOrBrokenError, "Unrecognized token type #{token_type}"
     end
   end
-end
+end

data/lib/hocon/impl/tokenizer.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'hocon/impl'
 require 'hocon/impl/config_impl_util'
 require 'hocon/impl/tokens'
@@ -7,6 +9,7 @@ require 'forwardable'
 
 class Hocon::Impl::Tokenizer
   Tokens = Hocon::Impl::Tokens
+  ConfigBugOrBrokenError = Hocon::ConfigError::ConfigBugOrBrokenError
 
   class TokenizerProblemError < StandardError
     def initialize(problem)
@@ -18,8 +21,36 @@ class Hocon::Impl::Tokenizer
     end
   end
 
+  def self.as_string(codepoint)
+    if codepoint == "\n"
+      "newline"
+    elsif codepoint == "\t"
+      "tab"
+    elsif codepoint == -1
+      "end of file"
+    elsif codepoint =~ /[[:cntrl:]]/
+      "control character 0x%x" % codepoint
+    else
+      "%c" % codepoint
+    end
+  end
+
+  # Tokenizes a Reader. Does not close the reader; you have to arrange to do
+  # that after you're done with the returned iterator.
+  def self.tokenize(origin, input, syntax)
+    TokenIterator.new(origin, input, syntax != Hocon::ConfigSyntax::JSON)
+  end
+
+  def self.render(tokens)
+    rendered_text = ""
+    while (t = tokens.next)
+      rendered_text << t.token_text
+    end
+    rendered_text
+  end
+
   class TokenIterator
-    extend Forwardable
+
     class WhitespaceSaver
       def initialize
         @whitespace = StringIO.new
@@ -27,17 +58,14 @@ class Hocon::Impl::Tokenizer
       end
 
       def add(c)
-        if @last_token_was_simple_value
-          @whitespace << c
-        end
+        @whitespace << c
       end
 
       def check(t, base_origin, line_number)
-        if Hocon::Impl::Tokenizer::TokenIterator.simple_value?(t)
+        if TokenIterator.simple_value?(t)
           next_is_a_simple_value(base_origin, line_number)
         else
-          next_is_not_a_simple_value
-          nil
+          next_is_not_a_simple_value(base_origin, line_number)
         end
       end
 
@@ -45,47 +73,71 @@ class Hocon::Impl::Tokenizer
       # called if the next token is not a simple value;
       # discards any whitespace we were saving between
       # simple values.
-      def next_is_not_a_simple_value
+      def next_is_not_a_simple_value(base_origin, line_number)
         @last_token_was_simple_value = false
-        @whitespace.reopen("")
+        create_whitespace_token_from_saver(base_origin, line_number)
       end
 
       # called if the next token IS a simple value,
       # so creates a whitespace token if the previous
       # token also was.
       def next_is_a_simple_value(base_origin, line_number)
-        if @last_token_was_simple_value
-          # need to save whitespace between the two so
-          # the parser has the option to concatenate it.
-          if @whitespace.length > 0
-            Tokens.new_unquoted_text(
-                line_origin(base_origin, line_number),
-                @whitespace.string = ""
-            )
-          end
-        end
+        t = create_whitespace_token_from_saver(base_origin, line_number)
+        @last_token_was_simple_value = true unless @last_token_was_simple_value
+        t
       end
 
+      def create_whitespace_token_from_saver(base_origin, line_number)
+        return nil unless @whitespace.length > 0
+        if (@last_token_was_simple_value)
+          t = Tokens.new_unquoted_text(
+              Hocon::Impl::Tokenizer::TokenIterator.line_origin(base_origin, line_number),
+              String.new(@whitespace.string)
+          )
+        else
+          t = Tokens.new_ignored_whitespace(
+              Hocon::Impl::Tokenizer::TokenIterator.line_origin(base_origin, line_number),
+              String.new(@whitespace.string)
+          )
+        end
+        @whitespace.string = ""
+        t
+      end
     end
 
-    # chars JSON allows a number to start with
-    FIRST_NUMBER_CHARS = "0123456789-"
-    # chars JSON allows to be part of a number
-    NUMBER_CHARS = "0123456789eE+-."
-    # chars that stop an unquoted string
-    NOT_IN_UNQUOTED_TEXT = "$\"{}[]:=,+#`^?!@*&\\"
+    def initialize(origin, input, allow_comments)
+      @origin = origin
+      @input = input
+      @allow_comments = allow_comments
+      @buffer = []
+      @line_number = 1
+      @line_origin = @origin.with_line_number(@line_number)
+      @tokens = []
+      @tokens << Tokens::START
+      @whitespace_saver = WhitespaceSaver.new
+    end
 
-    def self.problem(origin, what, message, suggest_quotes, cause)
-      if what.nil? || message.nil?
-        throw Hocon::ConfigError::ConfigBugOrBrokenError.new("internal error, creating bad TokenizerProblemError", nil)
+    # this should ONLY be called from nextCharSkippingComments
+    # or when inside a quoted string, or when parsing a sequence
+    # like ${ or +=, everything else should use
+    # nextCharSkippingComments().
+    def next_char_raw
+      if @buffer.empty?
+        begin
+          @input.readchar.chr
+        rescue EOFError
+          -1
+        end
+      else
+        @buffer.pop
       end
-      TokenizerProblemError.new(Tokens.new_problem(origin, what, message, suggest_quotes, cause))
     end
 
-    def self.simple_value?(t)
-      Tokens.substitution?(t) ||
-          Tokens.unquoted_text?(t) ||
-          Tokens.value?(t)
+    def put_back(c)
+      if @buffer.length > 2
+        raise ConfigBugOrBrokenError, "bug: putBack() three times, undesirable look-ahead"
+      end
+      @buffer.push(c)
     end
 
     def self.whitespace?(c)
@@ -96,20 +148,6 @@ class Hocon::Impl::Tokenizer
       (c != "\n") and (Hocon::Impl::ConfigImplUtil.whitespace?(c))
     end
 
-    def_delegator :@tokens, :each
-
-    def initialize(origin, input, allow_comments)
-      @origin = origin
-      @input = input
-      @allow_comments = allow_comments
-      @buffer = []
-      @line_number = 1
-      @line_origin = @origin.set_line_number(@line_number)
-      @tokens = []
-      @tokens << Tokens::START
-      @whitespace_saver = WhitespaceSaver.new
-    end
-
     def start_of_comment?(c)
       if c == -1
         false
@@ -133,25 +171,7 @@ class Hocon::Impl::Tokenizer
       end
     end
 
-    def put_back(c)
-      if @buffer.length > 2
-        raise ConfigBugError, "bug: putBack() three times, undesirable look-ahead"
-      end
-      @buffer.push(c)
-    end
-
-    def next_char_raw
-      if @buffer.empty?
-        begin
-          @input.readchar.chr
-        rescue EOFError
-          -1
-        end
-      else
-        @buffer.pop
-      end
-    end
-
+    # get next char, skipping non-newline whitespace
     def next_char_after_whitespace(saver)
       while true
         c = next_char_raw
@@ -167,6 +187,53 @@ class Hocon::Impl::Tokenizer
       end
     end
 
+    def self.problem(origin, what, message, suggest_quotes, cause)
+      if what.nil? || message.nil?
+        raise ConfigBugOrBrokenError.new("internal error, creating bad TokenizerProblemError")
+      end
+      TokenizerProblemError.new(Tokens.new_problem(origin, what, message, suggest_quotes, cause))
+    end
+
+    def self.line_origin(base_origin, line_number)
+      base_origin.with_line_number(line_number)
+    end
+
+    # ONE char has always been consumed, either the # or the first /, but not
+    # both slashes
+    def pull_comment(first_char)
+      double_slash = false
+      if first_char == '/'
+        discard = next_char_raw
+        if discard != '/'
+          raise ConfigBugOrBrokenError, "called pullComment but // not seen"
+        end
+        double_slash = true
+      end
+
+      io = StringIO.new
+      while true
+        c = next_char_raw
+        if (c == -1) || (c == "\n")
+          put_back(c)
+          if (double_slash)
+            return Tokens.new_comment_double_slash(@line_origin, io.string)
+          else
+            return Tokens.new_comment_hash(@line_origin, io.string)
+          end
+        else
+          io << c
+        end
+      end
+    end
+
+    # chars JSON allows a number to start with
+    FIRST_NUMBER_CHARS = "0123456789-"
+    # chars JSON allows to be part of a number
+    NUMBER_CHARS = "0123456789eE+-."
+    # chars that stop an unquoted string
+    NOT_IN_UNQUOTED_TEXT = "$\"{}[]:=,+#`^?!@*&\\"
+
+
     # The rules here are intended to maximize convenience while
     # avoiding confusion with real valid JSON. Basically anything
     # that parses as JSON is treated the JSON way and otherwise
@@ -209,27 +276,6 @@ class Hocon::Impl::Tokenizer
       Tokens.new_unquoted_text(origin, io.string)
     end
 
-
-    def pull_comment(first_char)
-      if first_char == '/'
-        discard = next_char_raw
-        if discard != '/'
-          raise ConfigBugError, "called pullComment but // not seen"
-        end
-      end
-
-      io = StringIO.new
-      while true
-        c = next_char_raw
-        if (c == -1) || (c == "\n")
-          put_back(c)
-          return Tokens.new_comment(@line_origin, io.string)
-        else
-          io << c
-        end
-      end
-    end
-
     def pull_number(first_char)
       sb = StringIO.new
       sb << first_char
@@ -251,17 +297,17 @@ class Hocon::Impl::Tokenizer
       begin
         if contained_decimal_or_e
           # force floating point representation
-          Tokens.new_double(@line_origin, s.to_f, s)
+          Tokens.new_double(@line_origin, Float(s), s)
         else
-          Tokens.new_long(@line_origin, s.to_i, s)
+          Tokens.new_long(@line_origin, Integer(s), s)
         end
       rescue ArgumentError => e
         if e.message =~ /^invalid value for (Float|Integer)\(\)/
           # not a number after all, see if it's an unquoted string.
-          s.each do |u|
-            if NOT_IN_UNQUOTED_TEXT.index
-              raise self.problem(@line_origin, u, "Reserved character '#{u}'" +
-                "is not allowed outside quotes", true, nil)
+          s.each_char do |u|
+            if NOT_IN_UNQUOTED_TEXT.index(u)
+              raise self.class.problem(@line_origin, u, "Reserved character '#{u}'" +
+                                                       "is not allowed outside quotes", true, nil)
             end
           end
           # no evil chars so we just decide this was a string and
@@ -273,25 +319,125 @@ class Hocon::Impl::Tokenizer
       end
     end
 
+    def pull_escape_sequence(sb, sb_orig)
+      escaped = next_char_raw
+
+      if escaped == -1
+        error_msg = "End of input but backslash in string had nothing after it"
+        raise self.class.problem(@line_origin, "", error_msg, false, nil)
+      end
+
+      # This is needed so we return the unescaped escape characters back out when rendering
+      # the token
+      sb_orig << "\\" << escaped
+
+      case escaped
+        when "\""
+          sb << "\""
+        when "\\"
+          sb << "\\"
+        when "/"
+          sb << "/"
+        when "b"
+          sb << "\b"
+        when "f"
+          sb << "\f"
+        when "n"
+          sb << "\n"
+        when "r"
+          sb << "\r"
+        when "t"
+          sb << "\t"
+        when "u"
+          codepoint = ""
+
+          # Grab the 4 hex chars for the unicode character
+          4.times do
+            c = next_char_raw
+
+            if c == -1
+              error_msg = "End of input but expecting 4 hex digits for \\uXXXX escape"
+              raise self.class.problem(@line_origin, c, error_msg, false, nil)
+            end
+
+            codepoint << c
+          end
+          sb_orig << codepoint
+          # Convert codepoint to a unicode character
+          packed = [codepoint.hex].pack("U")
+          if packed == "_"
+            raise self.class.problem(@line_origin, codepoint,
+                                     "Malformed hex digits after \\u escape in string: '#{codepoint}'",
+                                     false, nil)
+          end
+          sb << packed
+        else
+          error_msg = "backslash followed by '#{escaped}', this is not a valid escape sequence (quoted strings use JSON escaping, so use double-backslash \\ for literal backslash)"
+          raise self.class.problem(Hocon::Impl::Tokenizer.as_string(escaped), "", error_msg, false, nil)
+      end
+    end
+
+    def append_triple_quoted_string(sb, sb_orig)
+      # we are after the opening triple quote and need to consume the
+      # close triple
+      consecutive_quotes = 0
+
+      while true
+        c = next_char_raw
+
+        if c == '"'
+          consecutive_quotes += 1
+        elsif consecutive_quotes >= 3
+          # the last three quotes end the string and the other kept.
+          sb.string = sb.string[0...-3]
+          put_back c
+          break
+        else
+          consecutive_quotes = 0
+          if c == -1
+            error_msg = "End of input but triple-quoted string was still open"
+            raise self.class.problem(@line_origin, c, error_msg, false, nil)
+          elsif c == "\n"
+            # keep the line number accurate
+            @line_number += 1
+            @line_origin = @origin.with_line_number(@line_number)
+          end
+        end
+
+        sb << c
+        sb_orig << c
+      end
+    end
+
     def pull_quoted_string
       # the open quote has already been consumed
       sb = StringIO.new
+
+      # We need a second StringIO to keep track of escape characters.
+      # We want to return them exactly as they appeared in the original text,
+      # which means we will need a new StringIO to escape escape characters
+      # so we can also keep the actual value of the string. This is gross.
+      sb_orig = StringIO.new
+      sb_orig << '"'
+
       c = ""
       while c != '"'
         c = next_char_raw
         if c == -1
-          raise self.problem(@line_origin, c, "End of input but string quote was still open", false, nil)
+          raise self.class.problem(@line_origin, c, "End of input but string quote was still open", false, nil)
         end
 
         if c == "\\"
-          pull_escape_sequence(sb)
+          pull_escape_sequence(sb, sb_orig)
         elsif c == '"'
+          sb_orig << c
           # done!
         elsif c =~ /[[:cntrl:]]/
-          raise self.problem(@line_origin, c, "JSON does not allow unescaped #{c}" +
-                            " in quoted strings, use a backslash escape", false, nil)
+          raise self.class.problem(@line_origin, c, "JSON does not allow unescaped #{c}" +
+                                                   " in quoted strings, use a backslash escape", false, nil)
         else
           sb << c
+          sb_orig << c
         end
       end
 
@@ -299,13 +445,69 @@ class Hocon::Impl::Tokenizer
       if sb.length == 0
         third = next_char_raw
         if third == '"'
-          append_triple_quoted_string(sb)
+          sb_orig << third
+          append_triple_quoted_string(sb, sb_orig)
         else
           put_back(third)
         end
       end
 
-      Tokens.new_string(@line_origin, sb.string)
+      Tokens.new_string(@line_origin, sb.string, sb_orig.string)
+    end
+
+    def pull_plus_equals
+      # the initial '+' has already been consumed
+      c = next_char_raw
+
+      unless c == '='
+        error_msg = "'+' not followed by =, '#{c}' not allowed after '+'"
+        raise self.class.problem(@line_origin, c, error_msg, true, nil) # true = suggest quotes
+      end
+
+      Tokens::PLUS_EQUALS
+    end
+
+    def pull_substitution
+      # the initial '$' has already been consumed
+      c = next_char_raw
+      if c != '{'
+        error_msg = "'$' not followed by {, '#{c}' not allowed after '$'"
+        raise self.class.problem(@line_origin, c, error_msg, true, nil) # true = suggest quotes
+      end
+
+      optional = false
+      c = next_char_raw
+
+      if c == '?'
+        optional = true
+      else
+        put_back(c)
+      end
+
+      saver = WhitespaceSaver.new
+      expression = []
+
+      while true
+        t = pull_next_token(saver)
+        # note that we avoid validating the allowed tokens inside
+        # the substitution here; we even allow nested substitutions
+        # in the tokenizer. The parser sorts it out.
+
+        if t == Tokens::CLOSE_CURLY
+          # end the loop, done!
+          break
+        elsif t == Tokens::EOF
+          raise self.class.problem(@line_origin, t, "Substitution ${ was not closed with a }", false, nil)
+        else
+          whitespace = saver.check(t, @line_origin, @line_number)
+          unless whitespace.nil?
+            expression << whitespace
+          end
+          expression << t
+        end
+      end
+
+      Tokens.new_substitution(@line_origin, optional, expression)
     end
 
     def pull_next_token(saver)
@@ -316,7 +518,7 @@ class Hocon::Impl::Tokenizer
         # newline tokens have the just-ended line number
         line = Tokens.new_line(@line_origin)
         @line_number += 1
-        @line_origin = @origin.set_line_number(@line_number)
+        @line_origin = @origin.with_line_number(@line_number)
         line
       else
         t = nil
@@ -341,7 +543,7 @@ class Hocon::Impl::Tokenizer
             if FIRST_NUMBER_CHARS.index(c)
               t = pull_number(c)
             elsif NOT_IN_UNQUOTED_TEXT.index(c)
-              raise Hocon::Impl::Tokenizer::TokenIterator.problem(@line_origin, c, "Reserved character '#{c}' is not allowed outside quotes", true, nil)
+              raise self.class.problem(@line_origin, c, "Reserved character '#{c}' is not allowed outside quotes", true, nil)
             else
               put_back(c)
               t = pull_unquoted_text
@@ -350,13 +552,19 @@ class Hocon::Impl::Tokenizer
         end
 
         if t.nil?
-          raise ConfigBugError, "bug: failed to generate next token"
+          raise ConfigBugOrBrokenError, "bug: failed to generate next token"
         end
 
         t
       end
     end
 
+    def self.simple_value?(t)
+      Tokens.substitution?(t) ||
+          Tokens.unquoted_text?(t) ||
+          Tokens.value?(t)
+    end
+
     def queue_next_token
       t = pull_next_token(@whitespace_saver)
       whitespace = @whitespace_saver.check(t, @origin, @line_number)
@@ -366,6 +574,10 @@ class Hocon::Impl::Tokenizer
       @tokens.push(t)
     end
 
+    def has_next?
+      !@tokens.empty?
+    end
+
     def next
       t = @tokens.shift
       if (@tokens.empty?) and (t != Tokens::EOF)
@@ -375,19 +587,37 @@ class Hocon::Impl::Tokenizer
           @tokens.push(e.problem)
         end
         if @tokens.empty?
-          raise ConfigBugError, "bug: tokens queue should not be empty here"
+          raise ConfigBugOrBrokenError, "bug: tokens queue should not be empty here"
         end
       end
       t
     end
 
-    def empty?
-      @tokens.empty?
+    def remove
+      raise ConfigBugOrBrokenError, "Does not make sense to remove items from token stream"
     end
-  end
 
+    def each
+      while has_next?
+        # Have to use self.next instead of next because next is a reserved word
+        yield self.next
+      end
+    end
+
+    def map
+      token_list = []
+      each do |token|
+        # yield token to calling method, append whatever is returned from the
+        # map block to token_list
+        token_list << yield(token)
+      end
+      token_list
+    end
+
+    def to_list
+      # Return array of tokens from the iterator
+      self.map { |token| token }
+    end
 
-  def self.tokenize(origin, input, syntax)
-    TokenIterator.new(origin, input, syntax != Hocon::ConfigSyntax::JSON)
   end
-end
+end