bijou 0.1.0

data/lib/bijou/minicgi.rb

@@ -0,0 +1,159 @@
+#
+# Copyright (c) 2007-2008 Todd Lucas. All rights reserved.
+#
+# minicgi.rb - A lightweight version of Ruby's built-in CGI class
+#
+require 'cgi'
+require 'bijou/common'
+
+module Bijou
+  #
+  # MiniCGI provides some of the functionality of the Ruby CGI class in cgi.rb
+  # in a way that makes it accessible from the HttpRequest interface.
+  # The Ruby CGI class is multipurpose and provides featurs such as HTML 
+  # rendering, which isn't useful within the Bijou framework (in part, because
+  # it would bypass the Bijou output rendering mechanism). 
+  #
+  # One significant difference is that MiniCGI always preserves the query
+  # string values separately from any posted values. This is similar to the
+  # way that PHP and ASP work. Ruby CGI merges all paramters together,
+  # regardless of their origin. This feature is also useful, so MiniCGI 
+  # provides a merged parameter set as well, which is also available via
+  # the HttpRequest interface.
+  #
+  class MiniCGI
+    #
+    # Set single to true to have request parameters with single values 
+    # be converted from an array to a single value. Otherwise, all parameter
+    # values will be arrays.
+    #
+    def initialize(single = true)
+      @single = single
+
+      # Defines @params, @cookies, @multipart
+      extend ::CGI::QueryExtension
+      @multipart = false
+
+      # NOTE: We use alternate names for these variables (e.g., get instead
+      # of query_string) because CGI::QueryExtension takes the liberty of
+      # defining methods on this class for a large set of HTTP-oriented
+      # environment variables.
+      @get = nil
+      @post = nil
+      @server = nil
+      @method = nil # from REQUEST_METHOD
+
+      initialize_params()
+      initialize_server()
+    end
+
+    attr_reader :get, :post, :method, :server
+
+    def env_table 
+      ENV
+    end
+
+    def stdinput
+      $stdin
+    end
+
+    def stdoutput
+      $DEFAULT_OUTPUT
+    end
+
+    #
+    # Initialize the data from the query.
+    #
+    # NOTE: This overridden version of the CGI equivalent reads data into
+    # separate query_string and form tables and, optionally, changes the
+    # storage format of parameter values.
+    #
+    # Handles multipart forms (in particular, forms that involve file uploads).
+    # Reads query parameters into the @params field, and cookies into @cookies.
+    #
+    def initialize_params()
+      @method = env_table['REQUEST_METHOD']
+      @multipart = false
+
+      qs_params = ''
+      form_params = ''
+
+      case @method
+      when "GET", "HEAD", "POST"
+        if defined?(MOD_RUBY)
+          qs_params = (Apache::request.args or '')
+        else
+          qs_params = (env_table['QUERY_STRING'] or '')
+        end
+
+        if @method == 'POST'
+          if %r|\Amultipart/form-data.*boundary=\"?([^\";,]+)\"?|n.match(env_table['CONTENT_TYPE'])
+            boundary = $1.dup
+            @multipart = true
+            form_params = read_multipart(boundary, Integer(env_table['CONTENT_LENGTH']))
+          else
+            stdinput.binmode if defined? stdinput.binmode
+            form_params = (stdinput.read(Integer(env_table['CONTENT_LENGTH'])) or '')
+          end
+        end
+      else
+        qs_params = read_from_cmdline
+      end
+
+      query_string = ::CGI::parse(qs_params) || {}
+      form         = ::CGI::parse(form_params) || {}
+      cookies      = ::CGI::Cookie::parse((env_table['HTTP_COOKIE'] or 
+                                           env_table['COOKIE']))
+
+      # If requested, convert all single-element arrays to their values.
+      if @single
+        @get = Bijou::MiniCGI.singularize(query_string)
+        @post = Bijou::MiniCGI.singularize(form)
+        @cookies = Bijou::MiniCGI.singularize(cookies)
+      else
+        @get = query_string
+        @post = form
+        @cookies = cookies
+      end
+
+      @params = @post.clone
+      @params.merge!(@get)
+      @params.merge!(@cookies)
+    end
+
+    def initialize_server
+      @server = {}
+
+       %w[ AUTH_TYPE CONTENT_TYPE GATEWAY_INTERFACE PATH_INFO
+           PATH_TRANSLATED QUERY_STRING REMOTE_ADDR REMOTE_HOST
+           REMOTE_IDENT REMOTE_USER REQUEST_METHOD SCRIPT_NAME
+           SERVER_NAME SERVER_PROTOCOL SERVER_SOFTWARE
+
+           REQUEST_URI
+
+           HTTP_ACCEPT HTTP_ACCEPT_CHARSET HTTP_ACCEPT_ENCODING
+           HTTP_ACCEPT_LANGUAGE HTTP_CACHE_CONTROL HTTP_FROM HTTP_HOST
+           HTTP_NEGOTIATE HTTP_PRAGMA HTTP_REFERER HTTP_USER_AGENT 
+       ].each { |env|
+         @server[env] = env_table[env]
+       }
+    end
+
+    # Converts a hash of arrays to a hash of arrays and values whereby
+    # any single item arrays in the original hash become values in the 
+    # result hash.
+    def self.singularize(hash)
+      result = {}
+
+      # Extract any single-item values from their arrays.
+      hash.each {|k,v|
+        if v.length == 1
+          result[k] = v[0]
+        else
+          result[k] = v
+        end
+      }
+      return result
+    end
+  end
+end

data/lib/bijou/parser.rb

@@ -0,0 +1,1026 @@
+#
+# Copyright (c) 2007-2008 Todd Lucas. All rights reserved.
+#
+# parser.rb - The Bijou component parser
+#
+require 'bijou/common'
+require 'bijou/lexer'
+require 'bijou/backend'
+
+#--
+#
+# Use stringio, if available. Otherwise, use the alternative.
+#
+#++
+begin
+  require 'stringio'
+rescue LoadError
+  require 'bijou/prstringio'
+
+  class StringIO < PureRubyStringIO
+  end
+end
+
+#--
+# TODO:
+# Handle nested tags <% <% %> %> with an error
+# Disallow tags in defs (init/fini)
+#++
+
+#
+# The Bijou parser is a bi-modal top-down LL(1) parser. It is 
+# bi-modal in that it uses two lexers. One for markup and the 
+# other for Bijou tags.
+#
+class Bijou::Parser
+  include Bijou::Parse
+
+  # The filename is written to line marker comments in the output.
+  attr_reader :filename
+
+  attr_reader :diagnostics
+
+  def initialize
+    @debug = false
+    # REVIEW: Make this an option      
+    @strip = true
+    @strippingTag = false
+
+#    @debug = true
+    @diagnostics = nil
+    @input = nil
+    @backend = nil
+    detach_file
+  end
+
+  def detach_file
+    @filename = ''
+
+    # We keep the diagnostics and the backend
+    # @diagnostics
+    # @backend
+
+    @input = nil
+
+    @topLexer = nil
+    @tagLexer = nil
+
+    @lexer = nil
+    @parsingArgs = false
+    @namedSection = nil
+  end
+
+  def attach_file(file, filename)
+    @filename = filename
+
+    @diagnostics = Diagnostics.new
+
+    trace = false
+    use_markers = true # The Perl-oriented '#line' marker, for stack traces.
+
+    @input = LexerInput.new(file, @diagnostics)
+    @backend = Backend.new(@diagnostics, trace, use_markers)
+
+    @topLexer = TextLexer.new(@input)
+    @tagLexer = TagLexer.new(@input)
+
+    @lexer = @topLexer 
+    @parsingArgs = false
+    @namedSection = nil
+  end
+
+  def diagnostic(m, l, c)
+    if !l; l = @lexer.line; end
+    if !c; c = @lexer.column; end
+    m.at(l, c)
+    m
+  end
+
+  def message(s, line=nil, column=nil)
+    m = Message.new
+    m << s
+    @input.diagnostics.add_message(diagnostic(m, line, column))
+  end
+
+  def warning(s, line=nil, column=nil)
+    m = Warning.new
+    m << s
+    @input.diagnostics.add_warning(diagnostic(m, line, column))
+  end
+
+  def error(s, line=nil, column=nil)
+    m = Error.new
+    m << s
+    @input.diagnostics.add_error(diagnostic(m, line, column))
+  end
+
+  def debug_(str)
+    print str if @debug
+  end
+
+  def debug(str)
+    puts str if @debug
+  end
+
+  def assert(c)
+    if (!c)
+      error("assert failed on line " + @lexer.line.to_s)
+    end
+  end
+
+  def parseTagName
+    debug_("parseTagName: ")
+
+    # If the first character isn't a word character, return nil
+    tag = ''
+    while tok = @lexer.next_token
+      if tok != Token::Char || /\w/ !~ @lexer.text 
+        @lexer.pop_token
+        debug(tag)
+        return tag
+      end
+      tag << @lexer.text
+    end
+    debug(tag + "<eof>")
+  end
+
+  def eatWhitespace()
+    while tok = @lexer.next_token
+      if tok != Token::Char || @lexer.text !~ /\s/
+        @lexer.pop_token
+        return true
+      end
+    end
+    return nil
+  end
+
+  def eatLineWhitespace()
+    while tok = @lexer.next_token
+      if tok != Token::Char || @lexer.text !~ /\s/ || @lexer.text == '\n'
+        @lexer.pop_token
+        return true
+      end
+    end
+    return nil
+  end
+
+  def eatArgumentWhitespace()
+    lineComment = false
+
+    while tok = @lexer.next_token
+
+      if lineComment
+        if tok == Token::Char && @lexer.text == "\n"
+          # warning("eat arg crlf")
+          lineComment = false
+          next
+        else
+          # Eat comments
+          next
+        end
+      end
+
+      if tok == Token::Char
+        str = @lexer.text
+        
+        if str == '#'
+          # warning("eat arg comment")
+          lineComment = true
+        elsif str == "\n"
+          # warning("eat arg crlf")
+          # lineComment = false
+        elsif @lexer.text =~ /\s/
+          # Eat whitespace
+        else # non-whitespace
+          # This should be an identifier
+          # warning("eat arg ws 1 #{@lexer.text}")
+          @lexer.pop_token
+          return true
+        end
+      else
+        # warning("eat arg ws 3 #{@lexer.text} #{tok}")
+        @lexer.pop_token
+        return true
+      end
+    end
+
+    # EOF
+    return false
+  end
+
+  def parseCharSequence(re)
+    debug_("parseCharSequence: ")
+    name = ''
+
+    while tok = @lexer.next_token
+      text = @lexer.text
+      if tok == Token::Char && text =~ re
+        name << text
+      else
+        @lexer.pop_token
+        debug(name)
+        return name
+      end
+    end
+
+    debug(name + "<eof>")
+    return name
+  end
+
+  def parseIdentifier
+    parseCharSequence(/\w/)
+  end
+
+  def parseFilename
+    parseCharSequence(/[\\\/\w\.]/)
+  end
+
+  def expectChar(ch, msg='')
+    if tok = @lexer.next_token
+      if tok == Token::Char
+        if @lexer.text == '='
+          return true
+        end
+      end
+      error("expected #{ch}#{msg}")
+    else
+      error("expected #{ch}#{msg} at end of file")
+    end
+    false
+  end
+
+  def findTagClose(openToken, closeToken)
+    while tok = @lexer.next_token
+      if tok == Token::TagClose
+        tagClose = @lexer.text
+        if tagClose != closeToken
+          error("expected '#{closeToken}' to close '#{openToken}' tag")
+        end
+        return
+      end
+    end
+    error("expected close tag for '#{openToken}' at end of file")
+  end
+
+  # Returns a two item array as a result, the string following the pipe 
+  # operator '|' and a hash containing filters, if found.
+  def parseOutputFlags
+    result = ''
+    filters = []
+    while tok = @lexer.next_token
+      if tok == Token::Char
+        if @lexer.text =~ /[a-z]/
+          filters.push @lexer.text
+        elsif @lexer.text !~ /\s/
+          @lexer.pop_token
+          return [result, []]
+        end
+        result << @lexer.text
+      elsif tok == Token::TagClose
+        @lexer.pop_token
+        return [result, filters]
+      else
+        @lexer.pop_token
+        return [result, []]
+      end
+    end
+    error("expected '%>' to close output tag '<%=' at end of file")
+    return [result, []]
+  end
+  
+  # Parses named tags of the form <%name> and </%name>. Due to the ambiguity 
+  # that the start token causes between inline tags and named tags, this 
+  # routine performs a lookahead and dispatches to the correct parsing routine.
+  def parseNamedTag(tagStart)
+    debug("parseNamedTag #{tagStart}")
+    nameIdentifier = nil
+
+    if tagStart == '<%'
+      tagName = parseTagName
+      case tagName
+      when 'init'
+        @namedSection = tagName
+
+      when 'fini'
+        @namedSection = tagName
+
+      when 'method'
+        @namedSection = tagName
+
+      when 'args'
+        if @parsingArgs
+          error("args may not be nested within other args sections")
+        end
+        @parsingArgs = true
+      else
+        @lexer.start(tagStart, TagType::Inline)
+        parseInlineTag(tagStart)
+        return
+      end
+
+      @lexer.start(tagStart, TagType::Named)
+
+      start_line = @lexer.line
+      start_column = @lexer.column
+
+      if tagName == 'method'
+        # <%method name...
+        #         ^
+        debug_("ws(")
+        eatWhitespace
+        debug(")")
+
+        nameIdentifier = parseIdentifier
+        if nameIdentifier.empty?
+          error("expected method name")
+        end
+      end
+    else
+      assert(tagStart == '</%')
+      @lexer.start(tagStart, TagType::Named)
+
+      start_line = @lexer.line
+      start_column = @lexer.column
+
+      # Whitespace following this tag may be stripped
+      @strippingTag = true
+
+      tagName = parseTagName
+      if tagName
+        case tagName
+        when 'init'
+        when 'fini'
+        when 'args'
+          @parsingArgs = false
+        when 'method'
+        else
+          error("section close type '</%#{tagName}' is unrecognized")
+        end
+
+        @namedSection = nil
+      else
+        error("expected name after '</%'")
+      end
+    end
+
+    # <%name ...
+    #       ^
+    debug_("eatws(")
+    eatWhitespace
+    debug(")")
+
+    #
+    # NOTE: The token may be a Char, even though it's for a tag close.
+    #
+    # This is a special case for when a tag name goes against the close
+    # tag token (e.g., <%init>). Because of the ambiguity caused by the two 
+    # different tag types opening with the same token, we must lookahead while
+    # we parse the identifier. Since we haven't switched into TagType::Named
+    # yet, '>' is regarded as a Char instead of a TagClose.
+    # 
+
+    tok = @lexer.next_token
+    tagClose = @lexer.text
+
+    if tok == Token::TagClose || tok == Token::Char
+      if tagClose == '>'
+        # Send to the backend
+        if tagStart == '<%'
+          @backend.named_start_tag(tagName, nameIdentifier, 
+                                   start_line, start_column)
+        else
+          assert(tagStart == '</%')
+          @backend.named_end_tag(tagName)
+        end
+      else
+        error("expected '>' to close #{tagName} tag")
+      end
+    else
+      error("unexpected token '#{@lexer.text}' in " +
+            "#{tagName} tag", start_line, start_column)
+    end
+  end
+
+  # Parses tags of the form <% ... %>, which may contain any valid Ruby 
+  # statements. The contents of the inline tag are evaluated at runtime
+  # during the render phase. The puts and print methods may be used to
+  # render to the output stream.
+  def parseInlineTag(tagStart)
+    @lexer.start(tagStart, TagType::Inline)
+
+    debug("parseInlineTag #{tagStart}")
+
+    # BUGBUG: Turn off quote handling in comments.
+    # BUGBUG: We may want to turn of string parsing here (at the cost of not
+    # handling quoted close tokens).
+    start_line = @lexer.line
+    start_column = @lexer.column
+
+    result = ''
+
+    while tok = @lexer.next_token
+      if tok == Token::Char
+        result << @lexer.text
+      elsif tok == Token::String
+        result << @lexer.text
+      elsif tok == Token::TagClose
+        tagClose = @lexer.text
+        if tagClose != '%>'
+          error("expected '%>' to close inline '<%' tag at (#{start_line}, #{start_column})")
+        end
+
+        debug("#{result}%>")
+
+        # Send to the backend
+        @backend.inline_tag(result, @filename, start_line)
+        return
+      else
+        error("unexpected tag token '#{@lexer.text}'", start_line, start_column)
+      end
+    end
+
+    if @lexer.line != start_line
+      error("unclosed inline tag (was this supposed to be a named tag?)", 
+            start_line, start_column)
+    else
+      error("unclosed inline tag (was this supposed to be a named tag?)")
+    end
+  end
+
+  # Parses tags of the form <%= ... %> where the contents may be any valid
+  # Ruby expression. This expression is evaluated at runtime in a string
+  # context and the results are rendered to the output stream.
+  def parseOutputTag(tagStart)
+    @lexer.start(tagStart, TagType::Output)
+
+    debug("parseOutputTag #{tagStart}")
+
+    start_line = @lexer.line
+    start_column = @lexer.column
+
+    result = ''
+    filters = nil
+
+    while tok = @lexer.next_token
+      if tok == Token::Char
+        if @lexer.text == '|'
+          # Returns an array containing [string, array]
+          flags = parseOutputFlags
+          if flags[1].length > 0
+            # Flags were found.
+            filters = flags[1]
+          else
+            # False alarm
+            result << '|' + flags[0]
+          end
+        else
+          result << @lexer.text
+        end
+      elsif tok == Token::String
+        result << @lexer.text
+      elsif tok == Token::TagClose
+        tagClose = @lexer.text
+        if tagClose != '%>'
+          error("expected '%>' to close output '<%=' tag")
+        end
+
+        # Send to the backend
+        @backend.output_tag(result, filters, @filename, start_line)
+        return
+      else
+        error("invalid output tag syntax", start_line, start_column)
+        error("unexpected token '#{@lexer.text}'")
+      end
+    end
+
+    error("unclosed output tag")
+  end
+
+  # Parse until ',' or '&>' is found.
+  def parseCallArgValue
+    result = ''
+
+    start_line = @lexer.line
+    start_column = @lexer.column
+
+    while tok = @lexer.next_token
+      if tok == Token::Char
+        if @lexer.text == ','
+          @lexer.pop_token
+          return result.strip
+        end
+        result << @lexer.text
+      elsif tok == Token::String
+        result << @lexer.text
+      elsif tok == Token::Operator
+        # We have to include this in case it is within another hash.
+        result << @lexer.text
+      elsif tok == Token::TagClose
+        tagClose = @lexer.text
+        if tagClose != '&>'
+          error("expected '&>' to close call tag at (#{start_line}, #{start_column})")
+        end
+
+        @lexer.pop_token
+        return result.strip
+      else
+        error("unexpected token '#{@lexer.text}'", start_line, start_column)
+        return nil
+      end
+    end
+
+    error("unclosed call tag after assignment '=>'")
+    return nil
+  end
+
+  def parseCallArgList
+    debug("parseCallArgList")
+
+    start_line = @lexer.line
+    start_column = @lexer.column
+
+    arg_list = []
+    arg_name = nil
+    arg_value = nil
+
+    loop do
+      eatWhitespace
+      arg_name = parseFilename
+      if !arg_name || arg_name.empty?
+        error("Argument name expected")
+        return nil
+      end
+
+      eatWhitespace
+      if tok = @lexer.next_token
+        if tok == Token::Operator && @lexer.text == '=>'
+          arg_value = parseCallArgValue
+          debug "arg_value #{arg_value}"
+          if arg_value && !arg_value.empty?
+#            if arg_list.has_key?(arg_name)
+#              warning("argument '#{arg_name}' duplicated")
+#            end
+
+            arg_list.push [arg_name, arg_value]
+
+            arg_name = nil
+            arg_value = nil
+          else
+            error("empty argument value after assignment operator '=>'")
+            return nil
+          end
+          # look for , or %>
+        else
+          error("expected assignment operator '=>' after argument identifier #{@lexer.text}")
+          @lexer.pop_token
+          return nil
+        end
+      else
+        error("unclosed call tag")
+        return nil
+      end
+
+      eatWhitespace
+      if tok = @lexer.next_token
+        if tok == Token::TagClose
+          tagClose = @lexer.text
+          if tagClose == '&>'
+            # End of successful parse
+            @lexer.pop_token
+            return arg_list
+          else
+            error("expected '&>' to close call tag at (#{start_line}, #{start_column})")
+            @lexer.pop_token
+            return nil
+          end
+        elsif tok != Token::Char || @lexer.text != ','
+          error("unexpected token '#{@lexer.text}' after argument value", start_line, start_column)
+          @lexer.pop_token
+          # End of argument list
+          return nil
+        end
+      end
+
+    end
+  end
+
+  # Parses a tag of the form <& name { arg-list } &> where arg-list is
+  # an optional Ruby hash.
+  def parseCallTag(tagStart)
+    @lexer.start(tagStart, TagType::Call)
+
+    debug("parseCallTag #{tagStart}")
+
+    start_line = @lexer.line
+    start_column = @lexer.column
+
+    # Is this an indirect call?
+    indirect = false
+
+    if tok = @lexer.next_token
+      if tok == Token::Char && @lexer.text == '='
+        indirect = true
+      else
+        @lexer.pop_token
+      end
+    end
+
+    # REVIEW: Should we allow indirect calls with @member syntax?
+    eatWhitespace
+    callIdentifier = parseFilename
+    if !callIdentifier || callIdentifier.empty?
+      error("Identifier expected in call tag")
+    end
+
+    arg_list = nil
+
+    eatWhitespace
+    if tok = @lexer.next_token
+      if tok == Token::Char
+        if @lexer.text == ','
+          @lexer.tokenize_arguments = true
+          arg_list = parseCallArgList
+          @lexer.tokenize_arguments = false
+
+          findTagClose('<&', '&>')
+
+          if arg_list
+            # Send to the backend
+            @backend.call_tag(callIdentifier, arg_list, indirect, 
+                              @filename, start_line)
+          end
+
+          return
+        else
+          error("unexpected token '#{@lexer.text}' after call identifier", start_line, start_column)
+
+          findTagClose('<&', '&>')
+          return
+        end
+      else
+        tagClose = @lexer.text
+        if tagClose != '&>'
+          error("expected '&>' to close call tag at (#{start_line}, #{start_column})")
+        end
+
+        result = {}
+
+        # Send to the backend
+        @backend.call_tag(callIdentifier, result, indirect, 
+                          @filename, start_line)
+        return
+      end
+    end
+
+    error("unclosed call tag")
+    return
+
+
+    while tok = @lexer.next_token
+      if tok == Token::Char
+        result << @lexer.text
+      elsif tok == Token::String
+        result << @lexer.text
+      elsif tok == Token::TagClose
+        tagClose = @lexer.text
+        # BUGBUG <& X %> doesn't fire.
+        if tagClose != '&>'
+          error("expected '&>' to close call tag at (#{start_line}, #{start_column})")
+        end
+
+        if result.strip.empty?
+          result = '{}'
+        end
+
+        # Send to the backend
+        @backend.call_tag(callIdentifier, result, indirect, 
+                          @filename, start_line)
+        return
+      else
+        error("unexpected tag token '#{@lexer.text}'", start_line, start_column)
+      end
+    end
+
+    error("unclosed call tag")
+  end
+
+  # Parses tags of the form <%! name1="value1" name2 = "value2" ... %>, which
+  # control compile time and runtime behavior. These tags are usually placed
+  # near the top of the file.
+  def parseDirectiveTag(tagStart)
+    @lexer.start(tagStart, TagType::Directive)
+
+    start_line = @lexer.line
+    start_column = @lexer.column
+
+    here = 'in <%! directive'
+
+    errors = 0
+    directives = {}
+
+    while true
+      eatWhitespace
+
+      if tok = @lexer.next_token
+        if tok == Token::TagClose
+          if @lexer.text == '%>'
+            if directives.length > 0
+              # Send to the backend
+              @backend.directive_tag(directives, start_line, start_column)
+
+              # Whitespace following this tag may be stripped
+              @strippingTag = true
+            end
+          else
+            error("expected '%>' to close directive '<%!' tag")
+          end
+          return # Main loop exit
+        else
+          @lexer.pop_token
+        end
+      end
+      
+      identifierName = parseIdentifier
+      if identifierName.empty?
+        error("expected identifier in directive")
+      end
+
+      eatWhitespace
+      
+      if !expectChar('=', " after literal #{here}")
+        findTagClose('<%!', '%>')
+        return
+      end
+      
+      eatWhitespace
+      
+      if tok = @lexer.next_token
+        if tok == Token::String
+          directives[identifierName] = @lexer.text
+        else
+          error("unexpected token #{@lexer.text} after assignment #{here}")
+          findTagClose('<%!', '%>')
+          return
+        end
+      end
+    end
+
+  end
+
+  def parseArgsSection
+#    @lexer.start(tagStart, TagType::Call)
+
+    debug("parseArgsSection")
+
+    start_line = @lexer.line
+    start_column = @lexer.column
+
+    loop do
+      lineComment = false
+
+      # warning("before whitespace")
+      eatArgumentWhitespace
+      # warning("after whitespace")
+
+      tok = @lexer.next_token
+      if tok == Token::TagOpen
+        # Normal argument end case.
+        @lexer.pop_token
+        return
+      else
+        @lexer.pop_token
+      end
+
+      # warning('after whitespace')
+      argName = parseIdentifier
+      if argName.empty?
+        error('expected argument identifier')
+      end
+
+      # warning("after identifier '#{argName}'")
+      # Eat whitespace only up until a linefeed.
+      eatLineWhitespace
+
+      # NOTE: We require that any value specified be a single-line expression.
+      # Although the backend may be able to render a multi-line assignment,
+      # it is not possible to distinguish between a lone identifier and an
+      # expression continuation. (In addition, our lexer does not tokenize 
+      # identifiers, [the parser does], which would preclude a single-token 
+      # lookahead. This would only be useful if the assignment operator were 
+      # required.)
+
+      tok = @lexer.next_token
+      if tok == Token::TagOpen
+        # warning('args end 2')
+        # The last argument has no value.
+        @backend.add_argument(argName, nil, @filename,
+                              @lexer.line, @lexer.column)
+        @lexer.pop_token
+        return
+      elsif tok == Token::Operator && @lexer.text == '=>'
+        # warning('=>')
+
+        argValue = ''
+
+        while tok = @lexer.next_token
+          if tok == Token::Char
+            str = @lexer.text
+            if str == '#' || str == "\n"
+              @lexer.pop_token
+
+              argValue = argValue.strip
+              if argValue.empty?
+                error('An argument assignment has no value')
+              else
+                # warning "comment/eol #{argName} => #{argValue}"
+                @backend.add_argument(argName, argValue, @filename,
+                                      @lexer.line, @lexer.column)
+              end
+
+              argName = nil
+              argValue = ''
+              break
+            else
+              argValue << str
+            end
+          elsif tok == Token::String
+            argValue << @lexer.text
+          elsif tok == Token::TagOpen
+            # Syntax error of the form:
+            # arg-name => </%args>
+            error('An argument assignment has no value')
+            @lexer.pop_token
+            return
+          end
+        end
+      elsif tok == Token::Char && (@lexer.text == '#' || @lexer.text =~ /\w/)
+        # warning("comment or new identifier")
+        @backend.add_argument(argName, nil, @filename,
+                              @lexer.line, @lexer.column)
+        @lexer.pop_token
+      else
+        error("unexpected token '#{@lexer.text}' after argument '#{argName}'")
+        @lexer.pop_token
+
+        # Wait for the end token
+        while tok = @lexer.next_token
+          if tok == Token::TagOpen
+            @lexer.pop_token
+            return
+          end
+        end
+
+        return
+      end
+    end # while true
+
+  end
+
+  # This routine handles the non-markup tags that are meaningful to
+  # the  Bijou processor. It dispatches to the specific tag handlers.
+  # Parsing is done with the specialized tag lexer. Normal markup is
+  # parsed using a lexer that only recognizes Bijou tag open tokens.
+  # This bi-modal system allows us to ignore the complex parsing that
+  # would be required to handle HTML, scripts, style sheets, etc.
+  def parseTag(tagStart)
+    tagName = nil
+
+    @backend.tag_open(tagStart)
+
+    case tagStart
+    when '<%';
+      # This handler will determine whether it's named or inline.
+      parseNamedTag(tagStart)
+      return
+    when '<%='
+      parseOutputTag(tagStart)
+      return
+    when '<&';
+      parseCallTag(tagStart)
+      return
+    when '</%';
+      # This handler will determine whether it's named or inline.
+      parseNamedTag(tagStart)
+      return
+    when '<%!'
+      parseDirectiveTag(tagStart)
+      return
+    else;
+      raise "unexpected tag type #{tagStart}"
+    end
+  end
+
+  def render(component, source_filename=nil, cache_filename=nil, 
+             base=nil, require_list=nil)
+    @backend.render(component, source_filename, cache_filename, 
+                    base, require_list)
+  end
+
+  def parse(component, file, filename='')
+    if !file.respond_to?(:read)
+      if file.respond_to?(:to_s)
+        # This is a fallback; use StringIO for better performance.
+        file = StringIO.new(file.dup)
+      else
+        raise ArgumentError, "must be a file or a string"
+      end
+    end
+
+    parse_file(component, file, filename='')
+
+    render(component)
+  end
+
+  def parse_string(component, text)
+    file = StringIO.new(text.dup)
+    parse_file(component, file)
+    file.close
+  end
+
+  # The main top-level parsing loop looks for the special Bijou start tags,
+  # dispatches to the specialized parsing routines, and manages the lexer
+  # switching process.
+  def parse_file(component, file, filename='')
+    #
+    # Initialize parser members.
+    #
+    attach_file(file, filename)
+
+    #
+    # Main parsing loop
+    #
+    source_line = @lexer.line
+    buffer = ''
+    while tok = @lexer.next_token
+      if tok == Token::Char
+        buffer << @lexer.text
+      elsif tok == Token::TagOpen
+        if @strip && @strippingTag
+          buffer = stripLeader(buffer)
+        end
+        @strippingTag = false
+
+        @backend.markup_section(buffer, @filename, source_line)
+        buffer = ''
+
+        # NOTE: Care must be taked in the implementation of the parsing 
+        # routines because the lookahead token may buffer characters from
+        # the input stream. If this causes problems, then the lexer should
+        # should flush (push) the lookahead characters.
+        tagStart = @lexer.text
+
+        @lexer = @tagLexer
+        parseTag(tagStart)
+        @lexer = @topLexer
+
+        if @parsingArgs
+          @topLexer.tokenize_arguments = true
+          parseArgsSection
+          @topLexer.tokenize_arguments = false
+        end
+
+        source_line = @lexer.line
+      else
+        # puts "Token: " + @lexer.text
+        error("Unexpected token #{@lexer.text}")
+      end
+    end
+
+    if @parsingArgs
+      error("no end tag for args section at end of file")
+      @parsingArgs = false
+    end
+
+    # Final section
+    if !buffer.empty?
+      # The last blank lines are stripped
+      if @strip
+        buffer = stripTrailer(buffer)
+      end
+
+      @backend.markup_section(buffer, @filename, source_line)
+      buffer = ''
+    end
+
+    # print "Chars: " + @input.character.to_s
+    detach_file
+  end
+
+  # Whitespace formatting helper that removes a single leading blank line
+  # when preceded by a tag.
+  def stripLeader(buffer)
+    if buffer =~ /\A\s*?\n(.*)\Z/m
+      return $1
+    end
+    return buffer
+  end
+
+  # Whitespace formatting helper that removes any whitespace at the end of
+  # a component.
+  def stripTrailer(buffer)
+    if buffer =~ /\A(.*?)\s*\Z/m
+      return $1
+    end
+    return buffer
+  end
+end