typesafe_config 0.0.3 → 0.0.4

data/lib/typesafe/config/impl/tokenizer.rb

@@ -1,370 +0,0 @@
-require 'typesafe/config/impl'
-require 'typesafe/config/impl/config_impl_util'
-require 'typesafe/config/impl/tokens'
-require 'stringio'
-
-class Typesafe::Config::Impl::Tokenizer
-  Tokens = Typesafe::Config::Impl::Tokens
-
-  class TokenizerProblemError < StandardError
-  end
-
-  class TokenIterator
-    class WhitespaceSaver
-      def initialize
-        @whitespace = StringIO.new
-        @last_token_was_simple_value = false
-      end
-
-      def add(c)
-        if @last_token_was_simple_value
-          @whitespace << c
-        end
-      end
-
-      def check(t, base_origin, line_number)
-        if Typesafe::Config::Impl::Tokenizer::TokenIterator.simple_value?(t)
-          next_is_a_simple_value(base_origin, line_number)
-        else
-          next_is_not_a_simple_value
-          nil
-        end
-      end
-
-      private
-      # called if the next token is not a simple value;
-      # discards any whitespace we were saving between
-      # simple values.
-      def next_is_not_a_simple_value
-        @last_token_was_simple_value = false
-        @whitespace.reopen("")
-      end
-
-      # called if the next token IS a simple value,
-      # so creates a whitespace token if the previous
-      # token also was.
-      def next_is_a_simple_value(base_origin, line_number)
-        if @last_token_was_simple_value
-          # need to save whitespace between the two so
-          # the parser has the option to concatenate it.
-          if @whitespace.length > 0
-            Tokens.new_unquoted_text(
-                line_origin(base_origin, line_number),
-                @whitespace.string = ""
-            )
-          end
-        end
-      end
-
-    end
-
-    # chars JSON allows a number to start with
-    FIRST_NUMBER_CHARS = "0123456789-"
-    # chars JSON allows to be part of a number
-    NUMBER_CHARS = "0123456789eE+-."
-    # chars that stop an unquoted string
-    NOT_IN_UNQUOTED_TEXT = "$\"{}[]:=,+#`^?!@*&\\"
-
-    def self.simple_value?(t)
-      Tokens.substitution?(t) ||
-          Tokens.unquoted_text?(t) ||
-          Tokens.value?(t)
-    end
-
-    def self.whitespace?(c)
-      Typesafe::Config::Impl::ConfigImplUtil.whitespace?(c)
-    end
-
-    def self.whitespace_not_newline?(c)
-      (c != "\n") and (Typesafe::Config::Impl::ConfigImplUtil.whitespace?(c))
-    end
-
-    def initialize(origin, input, allow_comments)
-      @origin = origin
-      @input = input
-      @allow_comments = allow_comments
-      @buffer = []
-      @line_number = 1
-      @line_origin = @origin.set_line_number(@line_number)
-      @tokens = []
-      @tokens << Tokens::START
-      @whitespace_saver = WhitespaceSaver.new
-    end
-
-    def start_of_comment?(c)
-      if c == -1
-        false
-      else
-        if @allow_comments
-          if c == '#'
-            true
-          elsif c == '/'
-            maybe_second_slash = next_char_raw
-            # we want to predictably NOT consume any chars
-            put_back(maybe_second_slash)
-            if maybe_second_slash == '/'
-              true
-            else
-              false
-            end
-          end
-        else
-          false
-        end
-      end
-    end
-
-    def put_back(c)
-      if @buffer.length > 2
-        raise ConfigBugError, "bug: putBack() three times, undesirable look-ahead"
-      end
-      @buffer.push(c)
-    end
-
-    def next_char_raw
-      if @buffer.empty?
-        begin
-          @input.readchar
-        rescue EOFError
-          -1
-        end
-      else
-        @buffer.pop
-      end
-    end
-
-    def next_char_after_whitespace(saver)
-      while true
-        c = next_char_raw
-        if c == -1
-          return -1
-        else
-          if self.class.whitespace_not_newline?(c)
-            saver.add(c)
-          else
-            return c
-          end
-        end
-      end
-    end
-
-    # The rules here are intended to maximize convenience while
-    # avoiding confusion with real valid JSON. Basically anything
-    # that parses as JSON is treated the JSON way and otherwise
-    # we assume it's a string and let the parser sort it out.
-    def pull_unquoted_text
-      origin = @line_origin
-      io = StringIO.new
-      c = next_char_raw
-      while true
-        if (c == -1) or
-            (NOT_IN_UNQUOTED_TEXT.index(c)) or
-            (self.class.whitespace?(c)) or
-            (start_of_comment?(c))
-          break
-        else
-          io << c
-        end
-
-        # we parse true/false/null tokens as such no matter
-        # what is after them, as long as they are at the
-        # start of the unquoted token.
-        if io.length == 4
-          if io.string == "true"
-            return Tokens.new_boolean(origin, true)
-          elsif io.string == "null"
-            return Tokens.new_null(origin)
-          end
-        elsif io.length  == 5
-          if io.string == "false"
-            return Tokens.new_boolean(origin, false)
-          end
-        end
-
-        c = next_char_raw
-      end
-
-      # put back the char that ended the unquoted text
-      put_back(c)
-
-      Tokens.new_unquoted_text(origin, io.string)
-    end
-
-
-    def pull_comment(first_char)
-      if first_char == '/'
-        discard = next_char_raw
-        if discard != '/'
-          raise ConfigBugError, "called pullComment but // not seen"
-        end
-      end
-
-      io = StringIO.new
-      while true
-        c = next_char_raw
-        if (c == -1) || (c == "\n")
-          put_back(c)
-          return Tokens.new_comment(@line_origin, io.string)
-        else
-          io << c
-        end
-      end
-    end
-
-    def pull_number(first_char)
-      sb = StringIO.new
-      sb << first_char
-      contained_decimal_or_e = false
-      c = next_char_raw
-      while (c != -1) && (NUMBER_CHARS.index(c))
-        if (c == '.') ||
-            (c == 'e') ||
-            (c == 'E')
-          contained_decimal_or_e = true
-        end
-        sb << c
-        c = next_char_raw
-      end
-      # the last character we looked at wasn't part of the number, put it
-      # back
-      put_back(c)
-      s = sb.string
-      begin
-        if contained_decimal_or_e
-          # force floating point representation
-          Tokens.new_double(@line_origin, s.to_f, s)
-        else
-          Tokens.new_long(@line_origin, s.to_i, s)
-        end
-      rescue ArgumentError => e
-        if e.message =~ /^invalid value for (Float|Integer)\(\)/
-          # not a number after all, see if it's an unquoted string.
-          s.each do |u|
-            if NOT_IN_UNQUOTED_TEXT.index
-              raise problem(u, "Reserved character '#{u}'" +
-                "is not allowed outside quotes", true)
-            end
-          end
-          # no evil chars so we just decide this was a string and
-          # not a number.
-          Tokens.new_unquoted_text(@line_origin, s)
-        else
-          raise e
-        end
-      end
-    end
-
-    def pull_quoted_string
-      # the open quote has already been consumed
-      sb = StringIO.new
-      c = ""
-      while c != '"'
-        c = next_char_raw
-        if c == -1
-          raise problem("End of input but string quote was still open")
-        end
-
-        if c == "\\"
-          pull_escape_sequence(sb)
-        elsif c == '"'
-          # done!
-        elsif c =~ /[[:cntrl:]]/
-          raise problem(c, "JSON does not allow unescaped #{c}" +
-                            " in quoted strings, use a backslash escape")
-        else
-          sb << c
-        end
-      end
-
-      # maybe switch to triple-quoted string, sort of hacky...
-      if sb.length == 0
-        third = next_char_raw
-        if third == '"'
-          append_triple_quoted_string(sb)
-        else
-          put_back(third)
-        end
-      end
-
-      Tokens.new_string(@line_origin, sb.string)
-    end
-
-    def pull_next_token(saver)
-      c = next_char_after_whitespace(saver)
-      if c == -1
-        Tokens::EOF
-      elsif c == "\n"
-        # newline tokens have the just-ended line number
-        line = Tokens.new_line(@line_origin)
-        @line_number += 1
-        @line_origin = @origin.set_line_number(@line_number)
-        line
-      else
-        t = nil
-        if start_of_comment?(c)
-          t = pull_comment(c)
-        else
-          t = case c
-                when '"' then pull_quoted_string
-                when '$' then pull_substitution
-                when ':' then Tokens::COLON
-                when ',' then Tokens::COMMA
-                when '=' then Tokens::EQUALS
-                when '{' then Tokens::OPEN_CURLY
-                when '}' then Tokens::CLOSE_CURLY
-                when '[' then Tokens::OPEN_SQUARE
-                when ']' then Tokens::CLOSE_SQUARE
-                when '+' then pull_plus_equals
-                else nil
-              end
-
-          if t.nil?
-            if FIRST_NUMBER_CHARS.index(c)
-              t = pull_number(c)
-            elsif NOT_IN_UNQUOTED_TEXT.index(c)
-              raise problem(c, "Reserved character '#{c}' is not allowed outside quotes", true)
-            else
-              put_back(c)
-              t = pull_unquoted_text
-            end
-          end
-        end
-
-        if t.nil?
-          raise ConfigBugError, "bug: failed to generate next token"
-        end
-
-        t
-      end
-    end
-
-    def queue_next_token
-      t = pull_next_token(@whitespace_saver)
-      whitespace = @whitespace_saver.check(t, @origin, @line_number)
-      if whitespace
-        @tokens.push(whitespace)
-      end
-      @tokens.push(t)
-    end
-
-    def next
-      t = @tokens.shift
-      if (@tokens.empty?) and (t != Tokens::EOF)
-        begin
-          queue_next_token
-        rescue TokenizerProblemError => e
-          @tokens.push(e.problem)
-        end
-        if @tokens.empty?
-          raise ConfigBugError, "bug: tokens queue should not be empty here"
-        end
-      end
-      t
-    end
-  end
-
-
-  def self.tokenize(origin, input, syntax)
-    TokenIterator.new(origin, input, syntax != Typesafe::Config::ConfigSyntax::JSON)
-  end
-end

data/lib/typesafe/config/impl/tokens.rb

@@ -1,157 +0,0 @@
-require 'typesafe/config/impl'
-require 'typesafe/config/impl/token'
-require 'typesafe/config/impl/token_type'
-require 'typesafe/config/impl/config_number'
-require 'typesafe/config/impl/config_string'
-
-# FIXME the way the subclasses of Token are private with static isFoo and accessors is kind of ridiculous.
-class Typesafe::Config::Impl::Tokens
-  Token = Typesafe::Config::Impl::Token
-  TokenType = Typesafe::Config::Impl::TokenType
-  ConfigNumber = Typesafe::Config::Impl::ConfigNumber
-  ConfigString = Typesafe::Config::Impl::ConfigString
-
-  START = Token.new_without_origin(TokenType::START, "start of file")
-  EOF = Token.new_without_origin(TokenType::EOF, "end of file")
-  COMMA = Token.new_without_origin(TokenType::COMMA, "','")
-  EQUALS = Token.new_without_origin(TokenType::EQUALS, "'='")
-  COLON = Token.new_without_origin(TokenType::COLON, "':'")
-  OPEN_CURLY = Token.new_without_origin(TokenType::OPEN_CURLY, "'{'")
-  CLOSE_CURLY = Token.new_without_origin(TokenType::CLOSE_CURLY, "'}'")
-  OPEN_SQUARE = Token.new_without_origin(TokenType::OPEN_SQUARE, "'['")
-  CLOSE_SQUARE = Token.new_without_origin(TokenType::CLOSE_SQUARE, "']'")
-  PLUS_EQUALS = Token.new_without_origin(TokenType::PLUS_EQUALS, "'+='")
-
-  class Comment < Token
-    def initialize(origin, text)
-      super(TokenType::COMMENT, origin)
-      @text = text
-    end
-    attr_reader :text
-  end
-
-  # This is not a Value, because it requires special processing
-  class Substitution < Token
-    def initialize(origin, optional, expression)
-      super(TokenType::SUBSTITUTION, origin)
-      @optional = optional
-      @value = expression
-    end
-  end
-
-  class UnquotedText < Token
-    def initialize(origin, s)
-      super(TokenType::UNQUOTED_TEXT, origin)
-      @value = s
-    end
-    attr_reader :value
-
-    def to_s
-      "'#{value}'"
-    end
-  end
-
-  class Value < Token
-    def initialize(value)
-      super(TokenType::VALUE, value.origin)
-      @value = value
-    end
-    attr_reader :value
-
-    def to_s
-      "'#{value.unwrapped}' (#{Typesafe::Config::ConfigValueType.name(value.value_type)})"
-    end
-  end
-
-  class Line < Token
-    def initialize(origin)
-      super(TokenType::NEWLINE, origin)
-    end
-  end
-
-  class Problem < Token
-    def initialize(origin, what, message, suggest_quotes, cause)
-      super(TokenType::PROBLEM, origin)
-      @what = what
-      @message = message
-      @suggest_quotes = suggest_quotes
-      @cause = cause
-    end
-  end
-
-  def self.new_line(origin)
-    Line.new(origin)
-  end
-
-  def self.new_comment(origin, text)
-    Comment.new(origin, text)
-  end
-
-  def self.new_unquoted_text(origin, s)
-    UnquotedText.new(origin, s)
-  end
-
-  def self.new_value(value)
-    Value.new(value)
-  end
-
-  def self.new_string(origin, value)
-    new_value(ConfigString.new(origin, value))
-  end
-
-  def self.new_long(origin, value, original_text)
-    new_value(ConfigNumber.new_number(origin, value, original_text))
-  end
-
-  def self.comment?(t)
-    t.is_a?(Comment)
-  end
-
-  def self.comment_text(token)
-    if comment?(token)
-      token.text
-    else
-      raise ConfigBugError, "tried to get comment text from #{token}"
-    end
-  end
-
-  def self.substitution?(t)
-    t.is_a?(Substitution)
-  end
-
-  def self.unquoted_text?(token)
-    token.is_a?(UnquotedText)
-  end
-
-  def self.unquoted_text(token)
-    if unquoted_text?(token)
-      token.value
-    else
-      raise ConfigBugError, "tried to get unquoted text from #{token}"
-    end
-  end
-
-  def self.value?(token)
-    token.is_a?(Value)
-  end
-
-  def self.value(token)
-    if token.is_a?(Value)
-      token.value
-    else
-      raise ConfigBugError, "tried to get value of non-value token #{token}"
-    end
-  end
-
-  def self.value_with_type?(t, value_type)
-    value?(t) && (value(t).value_type == value_type)
-  end
-
-  def self.newline?(t)
-    t.is_a?(Line)
-  end
-
-  def self.problem?(t)
-    t.is_a?(Problem)
-  end
-end

data/lib/typesafe/config/impl/unmergeable.rb

@@ -1,4 +0,0 @@
-require 'typesafe/config/impl'
-
-module Typesafe::Config::Impl::Unmergeable
-end

data/lib/typesafe/config/impl.rb

@@ -1,5 +0,0 @@
-require 'typesafe/config'
-
-module Typesafe::Config::Impl
-
-end

data/lib/typesafe/config.rb

@@ -1,4 +0,0 @@
-require 'typesafe'
-
-module Typesafe::Config
-end

data/lib/typesafe.rb

@@ -1,2 +0,0 @@
-module Typesafe
-end