bijou 0.1.0

data/lib/bijou/lexer.rb

@@ -0,0 +1,513 @@
+#
+# Copyright (c) 2007-2008 Todd Lucas. All rights reserved.
+#
+# lexer.rb - The lexer classes used by the parser
+#
+require 'bijou/common'
+require 'bijou/diagnostics.rb'
+
+module Bijou
+  module Parse
+
+class Token
+  Null = 0
+  TagOpen = 1
+  TagClose = 2
+  Char = 3
+  String = 4
+  Operator = 5
+end
+
+#
+# Contains operations and state that is shared between the different lexers.
+# This simplifies the process of tracking the line and column numbers of the
+# input files. Also maintains a lookahead list of characters so the lexers
+# don't have to. This list is only used for named tags, which are reserved
+# words, so its length may be constrained.
+#
+class LexerInput
+  MaxLookahead = 15 # Must be > than max name in named tags (<%name>)
+
+  def initialize(file, diagnostics)
+    @diagnostics = diagnostics
+    @file = file
+    @character = 0
+    @line = 1
+    @column = 1
+    @lookahead = []
+    @columns = []
+  end
+
+  def close
+    @file.close
+  end
+
+  attr_reader :diagnostics, :file
+
+  def character=(n)
+    @character = n
+  end
+  def character
+    @character
+  end
+
+  def line=(n)
+    @line = n
+  end
+  def line
+    @line
+  end
+
+  def column
+    @column
+  end
+
+  def getc()
+    @character += 1
+    ch = @file.getc
+#puts "getc #{@character}: #{ch.chr}"
+    if ch == 10
+      @line += 1
+      @columns.push(@column)
+      if @columns.length > MaxLookahead
+        @columns.shift
+      end
+      @column = 1
+    elsif ch == 9
+      @column += 2 # REVIEW: We assume tabs of two spaces
+    else
+      @column += 1
+    end
+    @lookahead.push(ch)
+    if @lookahead.length > MaxLookahead 
+      @lookahead.shift 
+    end
+    return ch
+  end
+
+  def ungetc(ch)
+#puts "ungetc #{@character}: #{ch.chr}"
+    if ch == 10
+      @line -= 1
+      if @columns.length > 0
+        @column = @columns.pop
+      else
+        raise "column lookahead underflow"
+      end
+    end
+    @character -= 1
+    @file.ungetc(ch)
+  end
+  
+  # Pop a character from the lookahead list back to the file stream.
+  def pop
+    if @lookahead.length > 0
+      ungetc(@lookahead.pop)
+    else
+      raise "lookahead underflow"
+    end
+  end
+end
+
+#
+# The base class shared by the lexers. It encapsulates common functionality, 
+# such as token management and error reporting.
+#
+class Lexer
+  attr_reader :input
+
+  def initialize(input)
+    @input = input
+    @token = nil
+    @text = ''
+    # One token lookahead for LL(1)
+    @prev_token = nil
+    @prev_text = ''
+    @next_token = nil
+    @next_text = ''
+  end
+
+  def token
+    @token
+  end
+
+  def text
+    @text
+  end
+
+  # The current line number being scanned, used for diagnostics.
+  def line
+    @input.line
+  end
+
+  # The current column number being scanned, used for diagnostics.
+  def column
+    @input.column
+  end
+
+  # Returns the next character from the input stream.
+  def getc()
+    @input.getc
+  end
+
+  # Used for backtracking, puts the most recently removed character 
+  # back into the input stream.
+  def ungetc(ch)
+    @input.ungetc(ch)
+  end
+
+  # Used for backtracking, puts the most recently removed character 
+  # back into the input stream. The character was automatically 
+  # buffered when getc was called.
+  def pop()
+    @input.pop
+  end
+
+  def diagnostic(m, l, c)
+    if !l; l = line; end
+    if !c; c = column; end
+    m.at(l, c)
+    m
+  end
+
+  def warning(s, l=nil, c=nil)
+    m = Bijou::Parse::Warning.new
+    m << s
+    @input.diagnostics.add_warning(diagnostic(m, l, c))
+  end
+
+  def error(s, l=nil, c=nil)
+    m = Bijou::Parse::Error.new
+    m << s
+    @input.diagnostics.add_error(diagnostic(m, l, c))
+  end
+
+  def error(s)
+    m = Bijou::Parse::Error.new
+    m.at(line, column)
+    m << s
+    @input.diagnostics.add(m)
+  end
+
+  def warnings()
+    @input.diagnostics.warnings
+  end
+
+  def errors()
+    @input.diagnostics.errors
+  end
+
+  def push_token(token, text)
+    @prev_token = @token
+    @prev_text = @text
+    @token = token
+    @text = text
+    @next_token = nil
+    @next_text = ''
+    @token
+  end
+
+  # Used for backtracking by the one token
+  def pop_token
+    @next_token = @token
+    @next_text = @text
+    @token = @prev_token
+    @text = @prev_text
+    @prev_token = nil
+    @prev_text = ''
+    @token
+  end
+
+  # Used after a pop operation
+  def shift_token
+    push_token @next_token, @next_text
+  end
+
+  def prev_token
+    @prev_token
+  end
+
+  def peek_token
+    @next_token
+  end
+  
+  def is_token_buffered
+    @next_token ? true : false
+  end
+
+  def next_token
+    if is_token_buffered
+      shift_token
+    else
+      nil
+    end
+  end
+
+  def set_string_token(startToken, double)
+    start_line = @line
+    start_column = @column
+
+    buf = startToken # Raw string
+
+    while ch0 = getc
+      ch = ch0.chr
+      buf << ch
+
+      if ch == "\\"
+        # Ignore the next character, but preserve it.
+        ch1 = getc
+        buf << ch1.chr
+      elsif ch == '"'
+        if double
+          return push_token(Token::String, buf)
+        end
+      elsif ch == "'"
+        if !double
+          return push_token(Token::String, buf)
+        end
+      end
+#      push_token(Token::Char, buf)
+    end
+
+    warning("unterminated string literal", start_line, start_column)
+    return nil
+  end
+end
+
+class TextLexer < Lexer
+  attr_accessor :tokenize_arguments
+
+  def initialize(input)
+    super
+    @tokenize_arguments = false
+  end
+
+  def next_token()
+    if is_token_buffered
+      return shift_token
+    end
+
+    while ch0 = getc
+      ch = ch0.chr
+
+      if ch == '"'
+        if @tokenize_arguments
+          return set_string_token('"', true)
+        end
+      elsif ch == "'"
+        if @tokenize_arguments
+          return set_string_token("'", false)
+        end
+      elsif ch == '='
+        if @tokenize_arguments
+          ch1 = getc
+          if !ch1
+            # End of stream.
+            return push_token(Token::Char, ch)
+          elsif ch1.chr == '>'
+            return push_token(Token::Operator, "=>")
+          end
+        end
+      elsif ch == '<'
+        ch1 = getc
+        if !ch1
+          # End of stream.
+          return push_token(Token::Char, ch)
+        elsif ch1.chr == '%'
+          ch2 = getc
+          if ch2.chr == '='
+            return push_token(Token::TagOpen, "<%=")
+          elsif ch2.chr == '!'
+            return push_token(Token::TagOpen, "<%!")
+          end
+          ungetc(ch2)
+          return push_token(Token::TagOpen, "<%")
+        elsif ch1.chr == '&'
+          return push_token(Token::TagOpen, "<&")
+        elsif ch1.chr == '/'
+          ch2 = getc
+          if ch2.chr == '%'
+            return push_token(Token::TagOpen, "</%")
+          end
+          ungetc(ch2)
+        end
+        ungetc(ch1)
+      end
+      return push_token(Token::Char, ch)
+    end
+    nil
+  end
+end
+
+class TagType
+  Directive = 0  # <%! ... %>
+  Output = 1  # <%= ... %>
+  Inline = 2  # <% ... %>
+  Named = 3   # <%name ... > or </%name ... >
+  Call = 4    # <& name ... >
+end
+
+class TagLexer < Lexer
+  attr_accessor :tokenize_arguments
+
+  def initialize(input)
+    super(input)
+    @startToken = nil
+    @tagType = nil
+    @tokenize_arguments = false
+  end
+
+  def parse_string(stringType)
+    @lexer = @stringLexer
+    @lexer.type = stringType
+
+    while tok = @lexer.next_token
+      # print @lexer.text # print before switching lexers
+      if tok == Token::String
+        #          print "<" + @lexer.text + ">---"
+        #          print @lexer.text
+        @backend.tagString(@lexer.unquoted_string, @lexer.type)
+        return @lexer.raw_string
+      end
+    end
+  end
+
+  def start(tok, type)
+    @startToken = tok
+    @tagType = type
+  end
+
+  def next_token()
+    if is_token_buffered
+      return shift_token
+    end
+
+    while ch0 = getc
+      ch = ch0.chr
+      #puts "Read: '#{ch}' #{@tagType}"
+
+      if ch == '"'
+        return set_string_token('"', true)
+      elsif ch == "'"
+        return set_string_token("'", false)
+      elsif ch == '='
+        if @tokenize_arguments
+          ch1 = getc
+          if ch1.chr == '>'
+            return push_token(Token::Operator, "=>")
+          end
+        end
+      elsif ch == '%'
+        ch1 = getc
+        if ch1.chr == '>'
+#          if @startToken == '<%=' || @startToken == '<%' ||
+#             @startToken == '</%' || @startToken == '<%!'
+          return push_token(Token::TagClose, "%>")
+#          else
+#            warning("unexpected '%>' tag close")
+#          end
+        end
+        ungetc(ch1)
+      elsif ch == '&'
+        ch1 = getc
+        if ch1.chr == '>'
+#          if @startToken == '<&'
+          return push_token(Token::TagClose, "&>")
+#          else
+#            warning("unexpected '&>' tag close")
+#          end
+        end
+        ungetc(ch1)
+      elsif ch == '>'
+        if @tagType == TagType::Named || 
+           @tagType == TagType::Directive
+          return push_token(Token::TagClose, ">")
+        else
+          # For inline and calls, we ignore this case, because it may
+          # be '=>' '>=', etc., which are valid expressions.
+          # REVIEW: This can cause problems.
+          # BUGBUG: When parsing <%init>, for example, we haven't set the
+          # token type because we don't know it yet.
+#          print("#{@startToken} #{@tagType} ***")
+        end
+      end
+      return push_token(Token::Char, ch)
+    end
+    nil
+  end
+end
+
+#
+# The string lexer is used to switch the parser into a mode in which it can
+# intelligently ignore (or process) Ruby strings, including strings which 
+# have embedded tag close sequences like '>' and '%>'.
+#
+class StringLexer < Lexer
+  def initialize(input)
+    super(input)
+    @string = ""
+  end
+  
+  class Type
+    Single = 1
+    Double = 2
+  end
+
+  def type=(t)
+    @type = t
+    @string = '' # reinitialize
+  end
+
+  def type
+    @type
+  end
+
+  def unquoted_string
+    @string
+  end
+
+  def raw_string
+    if type == StringLexer::Type::Double
+      "\"#{@string}\""
+    else
+      "'#{@string}'"
+    end
+  end
+
+  def push_token(token, text)
+    if token == Token::Char
+#      print "(#{token}, #{text})"
+      @string << text
+    end
+    super(token, text)
+  end
+
+  def next_token()
+    if is_token_buffered
+      return shift_token
+    end
+
+    while ch0 = getc
+      ch = ch0.chr
+
+      if ch == "\\"
+        push_token(Token::Char, ch)
+        ch1 = getc
+        return push_token(Token::Char, ch1.chr)
+      elsif ch == '"'
+        if @type == Type::Double
+          return push_token(Token::String, ch)
+        end
+      elsif ch == "'"
+        if @type == Type::Single
+          return push_token(Token::String, ch)
+        end
+      end
+      return push_token(Token::Char, ch)
+    end
+    nil
+  end
+end
+
+  end # module Parse
+end   # module Bijou