sdl4r 0.9.1

data/lib/sdl4r/token.rb

@@ -0,0 +1,129 @@
+# Simple Declarative Language (SDL) for Ruby
+# Copyright 2005 Ikayzo, inc.
+#
+# This program is free software. You can distribute or modify it under the
+# terms of the GNU Lesser General Public License version 2.1 as published by
+# the Free Software Foundation.
+#
+# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
+# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, contact the Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+module SDL4R
+  
+  require File.dirname(__FILE__) + '/parser'
+
+  # An SDL token.
+  #
+  # @author Daniel Leuck, Philippe Vosges
+  #
+  class Token
+
+    def initialize(text, line = -1, position = -1)
+      @text = text
+      @line = line
+      @pos = position
+      @size = text.length
+
+      begin
+        @type = nil
+        @object = nil
+
+        if text =~ /^["`]/
+          @type = :STRING
+          @object = Parser.parse_string(text)
+
+        elsif text =~ /^'/
+          @type = :CHARACTER
+          @object = text[1...-1]
+
+        elsif text == "null"
+          @type = :NULL
+          @object = nil
+
+        elsif text =~ /^true$|^on$/
+          @type = :BOOLEAN
+          @object = true
+
+        elsif text =~ /^false$|^off$/
+          @type = :BOOLEAN
+          @object = false
+
+        elsif text =~ /^\[/
+          @type=:BINARY
+          @object = Parser.parse_binary(text)
+
+        elsif text =~ /^\d+\/\d+\/\d+$/
+          @type = :DATE;
+          @object = Parser.parse_date_time(text)
+
+        elsif text =~ /^-?\d+d?:\d+/
+          @type = :TIME
+          @object = parse_time_span_with_zone(text)
+
+        elsif text =~ /^[\d\-\.]/
+          @type = :NUMBER
+          @object = Parser.parse_number(text)
+
+        else
+          case text[0]
+          when ?{
+            @type = :START_BLOCK
+          when ?}
+            @type = :END_BLOCK
+          when ?=
+            @type = :EQUALS
+          when ?:
+            @type = :COLON
+          when ?;
+            @type = :SEMICOLON
+          end
+        end
+
+      rescue ArgumentError
+        raise SdlParseError.new($!.message, @line, @pos)
+      end
+
+      @type = :IDENTIFIER if @type.nil? # if all hope is lost, it's an identifier
+
+      @punctuation =
+        @type == :COLON || @type == :SEMICOLON || @type == :EQUALS ||
+        @type == :START_BLOCK || @type == :END_BLOCK
+      @literal = @type != :IDENTIFIER && !@punctuation
+    end
+
+    attr_reader :text, :type, :line, :position
+
+    def literal?
+      @literal
+    end
+
+    # Returns the Ruby object corresponding to this literal (or nil if it is
+    # not a literal).
+    def object_for_literal
+      return @object
+    end
+
+    def to_s
+      @type.to_s + " " + @text + " pos:" + @pos.to_s
+    end
+
+    # This special parse method is used only by the Token class for
+    # tokens which are ambiguously either a TimeSpan or the time component
+    # of a date/time type
+    def parse_time_span_with_zone(literal)
+      raise ArgumentError("time span or date literal is nil") if literal.nil?
+
+      days, hours, minutes, seconds, time_zone_offset =
+        Parser.parse_time_span_and_time_zone(literal, true, true)
+
+      return Parser::TimeSpanWithZone.new(days, hours, minutes, seconds, time_zone_offset)
+    end
+  end
+  
+end

data/lib/sdl4r/tokenizer.rb

@@ -0,0 +1,501 @@
+# Simple Declarative Language (SDL) for Ruby
+# Copyright 2005 Ikayzo, inc.
+#
+# This program is free software. You can distribute or modify it under the
+# terms of the GNU Lesser General Public License version 2.1 as published by
+# the Free Software Foundation.
+#
+# This program is distributed AS IS and WITHOUT WARRANTY. OF ANY KIND,
+# INCLUDING MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, contact the Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+module SDL4R
+
+  require File.dirname(__FILE__) + '/reader'
+  require File.dirname(__FILE__) + '/token'
+
+  # Tokenizer of the SDL parser
+  class Tokenizer
+
+    TOKEN_TYPES = [
+      :IDENTIFIER,
+
+      # punctuation
+      :COLON, :SEMICOLON, :EQUALS, :START_BLOCK, :END_BLOCK,
+
+      # literals
+      :STRING, :CHARACTER, :BOOLEAN, :NUMBER, :DATE, :TIME, :BINARY, :NULL ]
+
+    
+    # Creates an SDL tokenizer on the specified +IO+.
+    def initialize(io)
+      raise ArgumentError, "io == nil" if io.nil?
+      
+      @reader = Reader.new(io)
+      @token_start = 0
+      @startEscapedQuoteLine = false
+      @tokens = nil
+      @tokenText = nil
+    end
+
+    # Closes this Tokenizer and its underlying +Reader+.
+    def close
+      @reader.close
+    end
+
+    # Close the reader and throw a SdlParseError.
+    def parse_error(description, line_no = nil, position = nil)
+      begin
+        @reader.close()
+      rescue IOError
+        # no recourse
+      end
+
+      line_no = @reader.line_no if line_no.nil?
+      position = @reader.pos if position.nil?
+
+      # We add one because editors typically start with line 1 and position 1
+      # rather than 0...
+      raise SdlParseError.new(description, line_no + 1, position + 1, @reader.line)
+    end
+
+    # Close the reader and throw a SdlParseError using the format
+    # Was expecting X but got Y.
+    #
+    def expecting_but_got(expecting, got, line, position)
+      parse_error("Was expecting #{expecting} but got #{got}", line, position)
+    end
+
+    # Returns the next line as tokens or nil if the end of the stream has been reached.
+    # This method handles line continuations both within and outside String literals.
+    # The line of tokens is assigned to @tokens.
+    #
+    # Returns a logical line as a list of Tokens.
+    #
+    def read_line_tokens
+      begin
+        read_line_tokens_even_if_empty()
+      end until @tokens.nil? or !@tokens.empty?
+      return @tokens
+    end
+
+    def line_no
+      @reader.line_no
+    end
+
+    def pos
+      @reader.pos
+    end
+
+    def line
+      @reader.line
+    end
+
+    private
+
+    # Returns the next line as tokens or nil if the end of the stream has been reached.
+    # This method handles line continuations both within and outside String literals.
+    # The line of tokens is assigned to @tokens.
+    #
+    # Returns a logical line as a list of Tokens.
+    # Returns an empty array if the line was empty.
+    #
+    def read_line_tokens_even_if_empty
+      @tokens = nil
+      @tokenText = nil
+      @token_start = nil
+
+      @reader.read_line() if @reader.end_of_line?
+      return @tokens unless @reader.line
+
+      @tokens = []
+      @token_start = @reader.pos
+
+      while not @reader.end_of_line?
+        if @tokenText
+          @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
+          @tokenText = nil
+        end
+
+        c = @reader.current_char
+        next_c = @reader.get_line_char(@reader.pos + 1)
+        case c
+        when "\""
+          # handle "" style strings including line continuations
+          handle_double_quote_string()
+
+        when "'"
+          handle_character_literal()
+
+        when "{", "}", "=", ":", ";"
+          # handle punctuation
+          punctuation_token = Token.new(c, @reader.line_no, @reader.pos)
+          @tokenText = nil
+
+          if punctuation_token.type == :SEMICOLON
+            @reader.skip_char()
+            break
+          else
+            @tokens << punctuation_token
+          end
+
+        when "#"
+          # skip : hash comment at end of line
+          @reader.skip_line()
+
+        when "/"
+          # handle // and /**/ style comments
+          if next_c == "/"
+            # skip : // comment
+            @reader.skip_line()
+          else
+            handle_slash_comment()
+          end
+
+        when "`"
+          # handle multiline `` style strings
+          handle_back_quote_string()
+
+        when "["
+          # handle binary literals
+          handle_binary_literal()
+
+        when "\s", "\t"
+          @reader.skip_whitespaces()
+
+        when "\\"
+          # line continuations (outside a string literal)
+          handle_line_continuation();
+
+        when /^[0-9\-\.]$/
+          if c == "-" and next_c == "-"
+            # -- comments : ignore
+            @reader.skip_line()
+          else
+            # handle numbers, dates, and time spans
+            handle_number_date_or_time_span()
+          end
+
+        when /^[a-zA-Z\$_]$/
+          # FIXME Here, the Java code specifies isJavaIdentifierStart() but
+          # this is not easily implemented (at least as of Ruby 1.8).
+          # So, we implement a subset of these characters.
+          handle_identifier()
+
+        when "\n", "\r"
+          # end of line
+          @reader.skip_line()
+
+        else
+          parse_error("Unexpected character '#{c}'")
+        end
+
+        @reader.skip_char()
+      end
+
+      if @tokenText
+        @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
+      end
+
+      return @tokens
+    end
+
+    # Adds the current escaped character (represented by ((|c|))) to @tokenText.
+    # This method assumes the previous char was a backslash.
+    #
+    def add_escaped_char_in_string(c)
+      case c
+      when "\\", "\""
+        @tokenText << c
+      when "n"
+        @tokenText << ?\n
+      when "r"
+        @tokenText << ?\r
+      when "t"
+        @tokenText << ?\t
+      else
+        parse_error("Illegal escape character in string literal: '#{c.chr}'.")
+      end
+    end
+
+    def handle_double_quote_string
+      escaped = false
+      @startEscapedQuoteLine = false
+
+      @tokenText = "\""
+      @reader.skip_char()
+
+      while not @reader.end_of_line?
+        c = @reader.current_char
+
+        if "\s\t".include?(c) and @startEscapedQuoteLine
+          # we continue
+        else
+          @startEscapedQuoteLine = false;
+
+          if escaped
+            add_escaped_char_in_string(c)
+            escaped = false
+
+          elsif c == "\\"
+            # check for String broken across lines
+            if @reader.rest_of_line =~ /^\\\s*$/
+              handle_escaped_double_quoted_string()
+              next # as we are at the beginning of a new line
+            else
+              escaped = true;
+            end
+
+          else
+            @tokenText << c
+            if c == "\""
+              # end of double-quoted string detected
+              @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
+              @tokenText = nil
+              return
+            end
+          end
+        end
+
+        @reader.skip_char()
+      end
+
+      # detection of ill-terminated literals
+      if @tokenText =~ /^".*[^"]$/
+        parse_error(
+          "String literal \"#{@tokenText}\" not terminated by end quote.", @reader.line_no, @reader.line_length);
+      elsif @tokenText == "\""
+        parse_error("Orphan quote (unterminated string)", @reader.line_no, @reader.line_length);
+      end
+    end
+
+    def handle_escaped_double_quoted_string
+      # '\' can be followed by whitespaces
+      if @reader.rest_of_line =~ /^\\\s*$/
+        @reader.read_line()
+        parse_error("Escape at end of file.") if @reader.end_of_file?
+
+        @startEscapedQuoteLine = true
+
+      else
+        parse_error(
+          "Malformed string literal - escape followed by whitespace " +
+            "followed by non-whitespace.")
+      end
+    end
+
+    def handle_character_literal
+      if not @reader.more_chars_in_line?
+        parse_error("Got ' at end of line")
+      end
+
+      c2 = @reader.read_char()
+
+      if c2 == "\\"
+        if @reader.end_of_line?
+          parse_error("Got '\\ at end of line")
+        end
+
+        c3 = @reader.read_char()
+
+        if not @reader.more_chars_in_line?
+          parse_error("Got '\\#{c3} at end of line")
+        end
+
+        case c3
+        when "\\"
+          @tokens << Token.new("'\\'", @reader.line_no, @reader.pos)
+        when "'"
+          @tokens << Token.new("'''", @reader.line_no, @reader.pos)
+        when "n"
+          @tokens << Token.new("'\n'", @reader.line_no, @reader.pos)
+        when "r"
+          @tokens << Token.new("'\r'", @reader.line_no, @reader.pos)
+        when "t"
+          @tokens << Token.new("'\t'", @reader.line_no, @reader.pos)
+        else
+          parse_error("Illegal escape character #{@reader.current_char}")
+        end
+
+        @reader.skip_char()
+        if @reader.current_char != "'"
+          expecting_but_got("single quote (')", "\"#{@reader.current_char}\"")
+        end
+      else
+        @tokens << Token.new("'#{c2}'", @reader.line_no, @reader.pos)
+        if not @reader.more_chars_in_line?
+          parse_error("Got '#{c2} at end of line")
+        end
+        @reader.skip_char()
+        if @reader.current_char != "'"
+          expecting_but_got(
+            "quote (')", "\"#{@reader.current_char}\"", @reader.line_no, @reader.pos)
+        end
+      end
+    end
+
+    def handle_slash_comment
+      if not @reader.more_chars_in_line?
+        parse_error("Got slash (/) at end of line.")
+      end
+
+      if @reader.get_line_char(@reader.pos + 1) == "*"
+        end_index = @reader.find_next_in_line("*/")
+        if end_index
+          # handle comment on same line
+          @reader.skip_to(end_index + 1)
+        else
+          # handle multiline comments
+          loop do
+            @reader.read_raw_line()
+            if @reader.end_of_file?
+              parse_error("/* comment not terminated.", @reader.line_no, -2)
+            end
+
+            end_index = @reader.find_next_in_line("*/", 0)
+
+            if end_index
+              @reader.skip_to(end_index + 1)
+              break
+            end
+          end
+        end
+      elsif @reader.get_line_char(@reader.pos + 1) == "/"
+        parse_error("Got slash (/) in unexpected location.")
+      end
+    end
+
+    def handle_back_quote_string
+      end_index = @reader.find_next_in_line("`")
+
+      if end_index
+        # handle end quote on same line
+        @tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
+        @tokenText = nil
+        @reader.skip_to(end_index)
+
+      else
+        @tokenText = @reader.rest_of_line
+        @token_start = @reader.pos
+        # handle multiline quotes
+        loop do
+          @reader.read_raw_line()
+          if @reader.end_of_file?
+            parse_error("` quote not terminated.", @reader.line_no, -2)
+          end
+
+          end_index = @reader.find_next_in_line("`", 0)
+          if end_index
+            @tokenText << @reader.substring(0, end_index)
+            @reader.skip_to(end_index)
+            break
+          else
+            @tokenText << @reader.line
+          end
+        end
+
+        @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
+        @tokenText = nil
+      end
+    end
+
+    def handle_binary_literal
+      end_index = @reader.find_next_in_line("]")
+
+      if end_index
+        # handle end quote on same line
+        @tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
+        @tokenText = nil
+        @reader.skip_to(end_index)
+      else
+        @tokenText = @reader.substring(@reader.pos)
+        @token_start = @reader.pos
+        # handle multiline quotes
+        loop do
+          @reader.read_raw_line()
+          if @reader.end_of_file?
+            parse_error("[base64] binary literal not terminated.", @reader.line_no, -2)
+          end
+
+          end_index = @reader.find_next_in_line("]", 0)
+          if end_index
+            @tokenText << @reader.substring(0, end_index)
+            @reader.skip_to(end_index)
+            break
+          else
+            @tokenText << @reader.line
+          end
+        end
+
+        @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
+        @tokenText = nil
+      end
+    end
+
+    # handle a line continuation (not inside a string)
+    def handle_line_continuation
+      # backslash line continuation outside of a String literal
+      # can only occur at the end of a line
+      if not @reader.rest_of_line =~ /^\\\s*$/
+        parse_error("Line continuation (\\) before end of line")
+      else
+        @line = @reader.read_line()
+        if @line.nil?
+          parse_error("Line continuation at end of file.", @reader.line_no, @reader.pos)
+        end
+      end
+    end
+
+    def handle_number_date_or_time_span
+      @token_start = @reader.pos
+      @tokenText = ""
+
+      while not @reader.end_of_line?
+        c = @reader.current_char
+
+        if c =~ /[\w\.\-+:]/
+          @tokenText << c
+        elsif c == "/" and not @reader.get_line_char(@reader.pos + 1) == "*"
+          @tokenText << c
+        else
+          @reader.previous_char()
+          break
+        end
+
+        @reader.skip_char()
+      end
+
+      @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
+      @tokenText = nil
+    end
+
+    def handle_identifier
+      @token_start = @reader.pos;
+      @tokenText = ""
+
+      while not @reader.end_of_line?
+        c = @reader.current_char
+
+        # FIXME here we are stricter than the Java version because there is no
+        # easy way to implement Character.isJavaIdentifierPart() in Ruby :)
+        if c =~ /[\w_$-]/
+          @tokenText << c
+        else
+          @reader.previous_char()
+          break
+        end
+
+        @reader.skip_char()
+      end
+
+      @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
+      @tokenText = nil
+    end
+   
+  end
+end