typesafe_config 0.0.1

data/lib/typesafe/config/impl/tokenizer.rb

@@ -0,0 +1,370 @@
+require 'typesafe/config/impl'
+require 'typesafe/config/impl/config_impl_util'
+require 'typesafe/config/impl/tokens'
+require 'stringio'
+
+class Typesafe::Config::Impl::Tokenizer
+  Tokens = Typesafe::Config::Impl::Tokens
+
+  class TokenizerProblemError < StandardError
+  end
+
+  class TokenIterator
+    class WhitespaceSaver
+      def initialize
+        @whitespace = StringIO.new
+        @last_token_was_simple_value = false
+      end
+
+      def add(c)
+        if @last_token_was_simple_value
+          @whitespace << c
+        end
+      end
+
+      def check(t, base_origin, line_number)
+        if Typesafe::Config::Impl::Tokenizer::TokenIterator.simple_value?(t)
+          next_is_a_simple_value(base_origin, line_number)
+        else
+          next_is_not_a_simple_value
+          nil
+        end
+      end
+
+      private
+      # called if the next token is not a simple value;
+      # discards any whitespace we were saving between
+      # simple values.
+      def next_is_not_a_simple_value
+        @last_token_was_simple_value = false
+        @whitespace.reopen("")
+      end
+
+      # called if the next token IS a simple value,
+      # so creates a whitespace token if the previous
+      # token also was.
+      def next_is_a_simple_value(base_origin, line_number)
+        if @last_token_was_simple_value
+          # need to save whitespace between the two so
+          # the parser has the option to concatenate it.
+          if @whitespace.length > 0
+            Tokens.new_unquoted_text(
+                line_origin(base_origin, line_number),
+                @whitespace.string = ""
+            )
+          end
+        end
+      end
+
+    end
+
+    # chars JSON allows a number to start with
+    FIRST_NUMBER_CHARS = "0123456789-"
+    # chars JSON allows to be part of a number
+    NUMBER_CHARS = "0123456789eE+-."
+    # chars that stop an unquoted string
+    NOT_IN_UNQUOTED_TEXT = "$\"{}[]:=,+#`^?!@*&\\"
+
+    def self.simple_value?(t)
+      Tokens.substitution?(t) ||
+          Tokens.unquoted_text?(t) ||
+          Tokens.value?(t)
+    end
+
+    def self.whitespace?(c)
+      Typesafe::Config::Impl::ConfigImplUtil.whitespace?(c)
+    end
+
+    def self.whitespace_not_newline?(c)
+      (c != "\n") and (Typesafe::Config::Impl::ConfigImplUtil.whitespace?(c))
+    end
+
+    def initialize(origin, input, allow_comments)
+      @origin = origin
+      @input = input
+      @allow_comments = allow_comments
+      @buffer = []
+      @line_number = 1
+      @line_origin = @origin.set_line_number(@line_number)
+      @tokens = []
+      @tokens << Tokens::START
+      @whitespace_saver = WhitespaceSaver.new
+    end
+
+    def start_of_comment?(c)
+      if c == -1
+        false
+      else
+        if @allow_comments
+          if c == '#'
+            true
+          elsif c == '/'
+            maybe_second_slash = next_char_raw
+            # we want to predictably NOT consume any chars
+            put_back(maybe_second_slash)
+            if maybe_second_slash == '/'
+              true
+            else
+              false
+            end
+          end
+        else
+          false
+        end
+      end
+    end
+
+    def put_back(c)
+      if @buffer.length > 2
+        raise ConfigBugError, "bug: putBack() three times, undesirable look-ahead"
+      end
+      @buffer.push(c)
+    end
+
+    def next_char_raw
+      if @buffer.empty?
+        begin
+          @input.readchar
+        rescue EOFError
+          -1
+        end
+      else
+        @buffer.pop
+      end
+    end
+
+    def next_char_after_whitespace(saver)
+      while true
+        c = next_char_raw
+        if c == -1
+          return -1
+        else
+          if self.class.whitespace_not_newline?(c)
+            saver.add(c)
+          else
+            return c
+          end
+        end
+      end
+    end
+
+    # The rules here are intended to maximize convenience while
+    # avoiding confusion with real valid JSON. Basically anything
+    # that parses as JSON is treated the JSON way and otherwise
+    # we assume it's a string and let the parser sort it out.
+    def pull_unquoted_text
+      origin = @line_origin
+      io = StringIO.new
+      c = next_char_raw
+      while true
+        if (c == -1) or
+            (NOT_IN_UNQUOTED_TEXT.index(c)) or
+            (self.class.whitespace?(c)) or
+            (start_of_comment?(c))
+          break
+        else
+          io << c
+        end
+
+        # we parse true/false/null tokens as such no matter
+        # what is after them, as long as they are at the
+        # start of the unquoted token.
+        if io.length == 4
+          if io.string == "true"
+            return Tokens.new_boolean(origin, true)
+          elsif io.string == "null"
+            return Tokens.new_null(origin)
+          end
+        elsif io.length  == 5
+          if io.string == "false"
+            return Tokens.new_boolean(origin, false)
+          end
+        end
+
+        c = next_char_raw
+      end
+
+      # put back the char that ended the unquoted text
+      put_back(c)
+
+      Tokens.new_unquoted_text(origin, io.string)
+    end
+
+
+    def pull_comment(first_char)
+      if first_char == '/'
+        discard = next_char_raw
+        if discard != '/'
+          raise ConfigBugError, "called pullComment but // not seen"
+        end
+      end
+
+      io = StringIO.new
+      while true
+        c = next_char_raw
+        if (c == -1) || (c == "\n")
+          put_back(c)
+          return Tokens.new_comment(@line_origin, io.string)
+        else
+          io << c
+        end
+      end
+    end
+
+    def pull_number(first_char)
+      sb = StringIO.new
+      sb << first_char
+      contained_decimal_or_e = false
+      c = next_char_raw
+      while (c != -1) && (NUMBER_CHARS.index(c))
+        if (c == '.') ||
+            (c == 'e') ||
+            (c == 'E')
+          contained_decimal_or_e = true
+        end
+        sb << c
+        c = next_char_raw
+      end
+      # the last character we looked at wasn't part of the number, put it
+      # back
+      put_back(c)
+      s = sb.string
+      begin
+        if contained_decimal_or_e
+          # force floating point representation
+          Tokens.new_double(@line_origin, s.to_f, s)
+        else
+          Tokens.new_long(@line_origin, s.to_i, s)
+        end
+      rescue ArgumentError => e
+        if e.message =~ /^invalid value for (Float|Integer)\(\)/
+          # not a number after all, see if it's an unquoted string.
+          s.each do |u|
+            if NOT_IN_UNQUOTED_TEXT.index
+              raise problem(u, "Reserved character '#{u}'" +
+                "is not allowed outside quotes", true)
+            end
+          end
+          # no evil chars so we just decide this was a string and
+          # not a number.
+          Tokens.new_unquoted_text(@line_origin, s)
+        else
+          raise e
+        end
+      end
+    end
+
+    def pull_quoted_string
+      # the open quote has already been consumed
+      sb = StringIO.new
+      c = ""
+      while c != '"'
+        c = next_char_raw
+        if c == -1
+          raise problem("End of input but string quote was still open")
+        end
+
+        if c == "\\"
+          pull_escape_sequence(sb)
+        elsif c == '"'
+          # done!
+        elsif c =~ /[[:cntrl:]]/
+          raise problem(c, "JSON does not allow unescaped #{c}" +
+                            " in quoted strings, use a backslash escape")
+        else
+          sb << c
+        end
+      end
+
+      # maybe switch to triple-quoted string, sort of hacky...
+      if sb.length == 0
+        third = next_char_raw
+        if third == '"'
+          append_triple_quoted_string(sb)
+        else
+          put_back(third)
+        end
+      end
+
+      Tokens.new_string(@line_origin, sb.string)
+    end
+
+    def pull_next_token(saver)
+      c = next_char_after_whitespace(saver)
+      if c == -1
+        Tokens::EOF
+      elsif c == "\n"
+        # newline tokens have the just-ended line number
+        line = Tokens.new_line(@line_origin)
+        @line_number += 1
+        @line_origin = @origin.set_line_number(@line_number)
+        line
+      else
+        t = nil
+        if start_of_comment?(c)
+          t = pull_comment(c)
+        else
+          t = case c
+                when '"' then pull_quoted_string
+                when '$' then pull_substitution
+                when ':' then Tokens::COLON
+                when ',' then Tokens::COMMA
+                when '=' then Tokens::EQUALS
+                when '{' then Tokens::OPEN_CURLY
+                when '}' then Tokens::CLOSE_CURLY
+                when '[' then Tokens::OPEN_SQUARE
+                when ']' then Tokens::CLOSE_SQUARE
+                when '+' then pull_plus_equals
+                else nil
+              end
+
+          if t.nil?
+            if FIRST_NUMBER_CHARS.index(c)
+              t = pull_number(c)
+            elsif NOT_IN_UNQUOTED_TEXT.index(c)
+              raise problem(c, "Reserved character '#{c}' is not allowed outside quotes", true)
+            else
+              put_back(c)
+              t = pull_unquoted_text
+            end
+          end
+        end
+
+        if t.nil?
+          raise ConfigBugError, "bug: failed to generate next token"
+        end
+
+        t
+      end
+    end
+
+    def queue_next_token
+      t = pull_next_token(@whitespace_saver)
+      whitespace = @whitespace_saver.check(t, @origin, @line_number)
+      if whitespace
+        @tokens.push(whitespace)
+      end
+      @tokens.push(t)
+    end
+
+    def next
+      t = @tokens.shift
+      if (@tokens.empty?) and (t != Tokens::EOF)
+        begin
+          queue_next_token
+        rescue TokenizerProblemError => e
+          @tokens.push(e.problem)
+        end
+        if @tokens.empty?
+          raise ConfigBugError, "bug: tokens queue should not be empty here"
+        end
+      end
+      t
+    end
+  end
+
+
+  def self.tokenize(origin, input, syntax)
+    TokenIterator.new(origin, input, syntax != Typesafe::Config::ConfigSyntax::JSON)
+  end
+end

data/lib/typesafe/config/impl/tokens.rb

@@ -0,0 +1,157 @@
+require 'typesafe/config/impl'
+require 'typesafe/config/impl/token'
+require 'typesafe/config/impl/token_type'
+require 'typesafe/config/impl/config_number'
+require 'typesafe/config/impl/config_string'
+
+# FIXME the way the subclasses of Token are private with static isFoo and accessors is kind of ridiculous.
+class Typesafe::Config::Impl::Tokens
+  Token = Typesafe::Config::Impl::Token
+  TokenType = Typesafe::Config::Impl::TokenType
+  ConfigNumber = Typesafe::Config::Impl::ConfigNumber
+  ConfigString = Typesafe::Config::Impl::ConfigString
+
+  START = Token.new_without_origin(TokenType::START, "start of file")
+  EOF = Token.new_without_origin(TokenType::EOF, "end of file")
+  COMMA = Token.new_without_origin(TokenType::COMMA, "','")
+  EQUALS = Token.new_without_origin(TokenType::EQUALS, "'='")
+  COLON = Token.new_without_origin(TokenType::COLON, "':'")
+  OPEN_CURLY = Token.new_without_origin(TokenType::OPEN_CURLY, "'{'")
+  CLOSE_CURLY = Token.new_without_origin(TokenType::CLOSE_CURLY, "'}'")
+  OPEN_SQUARE = Token.new_without_origin(TokenType::OPEN_SQUARE, "'['")
+  CLOSE_SQUARE = Token.new_without_origin(TokenType::CLOSE_SQUARE, "']'")
+  PLUS_EQUALS = Token.new_without_origin(TokenType::PLUS_EQUALS, "'+='")
+
+  class Comment < Token
+    def initialize(origin, text)
+      super(TokenType::COMMENT, origin)
+      @text = text
+    end
+    attr_reader :text
+  end
+
+  # This is not a Value, because it requires special processing
+  class Substitution < Token
+    def initialize(origin, optional, expression)
+      super(TokenType::SUBSTITUTION, origin)
+      @optional = optional
+      @value = expression
+    end
+  end
+
+  class UnquotedText < Token
+    def initialize(origin, s)
+      super(TokenType::UNQUOTED_TEXT, origin)
+      @value = s
+    end
+    attr_reader :value
+
+    def to_s
+      "'#{value}'"
+    end
+  end
+
+  class Value < Token
+    def initialize(value)
+      super(TokenType::VALUE, value.origin)
+      @value = value
+    end
+    attr_reader :value
+
+    def to_s
+      "'#{value.unwrapped}' (#{Typesafe::Config::ConfigValueType.name(value.value_type)})"
+    end
+  end
+
+  class Line < Token
+    def initialize(origin)
+      super(TokenType::NEWLINE, origin)
+    end
+  end
+
+  class Problem < Token
+    def initialize(origin, what, message, suggest_quotes, cause)
+      super(TokenType::PROBLEM, origin)
+      @what = what
+      @message = message
+      @suggest_quotes = suggest_quotes
+      @cause = cause
+    end
+  end
+
+  def self.new_line(origin)
+    Line.new(origin)
+  end
+
+  def self.new_comment(origin, text)
+    Comment.new(origin, text)
+  end
+
+  def self.new_unquoted_text(origin, s)
+    UnquotedText.new(origin, s)
+  end
+
+  def self.new_value(value)
+    Value.new(value)
+  end
+
+  def self.new_string(origin, value)
+    new_value(ConfigString.new(origin, value))
+  end
+
+  def self.new_long(origin, value, original_text)
+    new_value(ConfigNumber.new_number(origin, value, original_text))
+  end
+
+  def self.comment?(t)
+    t.is_a?(Comment)
+  end
+
+  def self.comment_text(token)
+    if comment?(token)
+      token.text
+    else
+      raise ConfigBugError, "tried to get comment text from #{token}"
+    end
+  end
+
+  def self.substitution?(t)
+    t.is_a?(Substitution)
+  end
+
+  def self.unquoted_text?(token)
+    token.is_a?(UnquotedText)
+  end
+
+  def self.unquoted_text(token)
+    if unquoted_text?(token)
+      token.value
+    else
+      raise ConfigBugError, "tried to get unquoted text from #{token}"
+    end
+  end
+
+  def self.value?(token)
+    token.is_a?(Value)
+  end
+
+  def self.value(token)
+    if token.is_a?(Value)
+      token.value
+    else
+      raise ConfigBugError, "tried to get value of non-value token #{token}"
+    end
+  end
+
+  def self.value_with_type?(t, value_type)
+    value?(t) && (value(t).value_type == value_type)
+  end
+
+  def self.newline?(t)
+    t.is_a?(Line)
+  end
+
+  def self.problem?(t)
+    t.is_a?(Problem)
+  end
+end

data/lib/typesafe/config/impl/unmergeable.rb

@@ -0,0 +1,4 @@
+require 'typesafe/config/impl'
+
+module Typesafe::Config::Impl::Unmergeable
+end

data/lib/typesafe/config/impl.rb

@@ -0,0 +1,5 @@
+require 'typesafe/config'
+
+module Typesafe::Config::Impl
+
+end

data/lib/typesafe/config.rb

@@ -0,0 +1,4 @@
+require 'typesafe'
+
+module Typesafe::Config
+end

data/lib/typesafe.rb

@@ -0,0 +1,2 @@
+module Typesafe
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: typesafe_config
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Chris Price
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-14 00:00:00.000000000 Z
+dependencies: []
+description: ==A port of the Java [Typesafe Config](https://github.com/typesafehub/config)
+  library to Ruby
+email: chris@puppetlabs.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/typesafe/config.rb
+- lib/typesafe/config/config_object.rb
+- lib/typesafe/config/config_error.rb
+- lib/typesafe/config/impl/simple_includer.rb
+- lib/typesafe/config/impl/config_number.rb
+- lib/typesafe/config/impl/simple_config.rb
+- lib/typesafe/config/impl/config_int.rb
+- lib/typesafe/config/impl/resolve_status.rb
+- lib/typesafe/config/impl/simple_config_origin.rb
+- lib/typesafe/config/impl/token.rb
+- lib/typesafe/config/impl/config_string.rb
+- lib/typesafe/config/impl/config_float.rb
+- lib/typesafe/config/impl/full_includer.rb
+- lib/typesafe/config/impl/parser.rb
+- lib/typesafe/config/impl/tokens.rb
+- lib/typesafe/config/impl/tokenizer.rb
+- lib/typesafe/config/impl/abstract_config_object.rb
+- lib/typesafe/config/impl/simple_include_context.rb
+- lib/typesafe/config/impl/token_type.rb
+- lib/typesafe/config/impl/path_builder.rb
+- lib/typesafe/config/impl/parseable.rb
+- lib/typesafe/config/impl/path.rb
+- lib/typesafe/config/impl/unmergeable.rb
+- lib/typesafe/config/impl/config_impl_util.rb
+- lib/typesafe/config/impl/simple_config_object.rb
+- lib/typesafe/config/impl/simple_config_list.rb
+- lib/typesafe/config/impl/config_impl.rb
+- lib/typesafe/config/impl/config_concatenation.rb
+- lib/typesafe/config/impl/abstract_config_value.rb
+- lib/typesafe/config/impl/origin_type.rb
+- lib/typesafe/config/config_factory.rb
+- lib/typesafe/config/impl.rb
+- lib/typesafe/config/config_parse_options.rb
+- lib/typesafe/config/config_render_options.rb
+- lib/typesafe/config/config_value_type.rb
+- lib/typesafe/config/config_syntax.rb
+- lib/typesafe.rb
+- LICENSE
+- README.md
+homepage: https://github.com/cprice404/ruby-typesafe-config
+licenses:
+- Apache License, v2
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Typesafe Config
+test_files: []