taskjuggler 0.0.6 → 0.0.7

data/lib/RTFWithQuerySupport.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+#
+# = RTFWithQuerySupport.rb -- The TaskJuggler III Project Management Software
+#
+# Copyright (c) 2006, 2007, 2008, 2009, 2010 by Chris Schlaeger <cs@kde.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+
+require 'RichTextFunctionHandler'
+
+class TaskJuggler
+
+  class RTFWithQuerySupport < RichTextFunctionHandler
+
+    def initialize(messageHandler, type, sourceFileInfo = nil)
+      super
+      @query = nil
+    end
+
+    # This function must be called to register the Query object that will be
+    # used to resolve the queries. It will create a copy of the object since
+    # it will modify it.
+    def setQuery(query)
+      @query = query.dup
+    end
+
+  end
+
+  class RichTextIntermediate
+
+    def setQuery(query)
+      @functionHandlers.each_value do |handler|
+        if handler.respond_to?('setQuery')
+          handler.setQuery(query)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/Resource.rb

@@ -63,14 +63,18 @@ class TaskJuggler
       (numberOfLevels - 1).downto(0) do |level|
         levelList = listByLevel[level]
         levelRecord = @project['alertLevels'][level]
-        alertName = "[[File:icons/flag-#{levelRecord[0]}.png|" +
-                    "alt=[#{levelRecord[1]}]|text-bottom]]"
+        if query.selfContained
+          alertName = "<nowiki>[#{levelRecord[1]}]</nowiki>"
+        else
+          alertName = "[[File:icons/flag-#{levelRecord[0]}.png|" +
+          "alt=[#{levelRecord[1]}]|text-bottom]]"
+        end
         levelList.each do |entry|
           tsRecord = entry.timeSheetRecord
 
           if entry.property.is_a?(Task)
-            rText += "== #{alertName} <nowiki>#{entry.property.name}</nowiki> "+
-              "(ID: #{entry.property.fullId}) ==\n\n"
+            rText += "=== #{alertName} <nowiki>#{entry.property.name}" +
+                     "</nowiki> (ID: #{entry.property.fullId}) ===\n\n"
             if tsRecord
               rText += "'''Work:''' #{tsRecord.actualWorkPercent.to_i}% "
               if tsRecord.actualWorkPercent != tsRecord.planWorkPercent
@@ -122,13 +126,11 @@ class TaskJuggler
 
       # Now convert the RichText markup String into RichTextIntermediate
       # format.
-      begin
-        rti = RichText.new(rText, RTFHandlers.create(@project)).
-          generateIntermediateFormat
-      rescue RichTextException => msg
-        $stderr.puts "Error while processing Rich Text\n" +
-                     "Line #{msg.lineNo}: #{msg.text}\n" +
-                     "#{msg.line}"
+      unless (rti = RichText.new(rText, RTFHandlers.create(@project),
+                                 @project.messageHandler).
+                                 generateIntermediateFormat)
+        @project.messageHandler.warning(
+          'res_journal_text', 'Syntax error in journal text')
         return nil
       end
       # No section numbers, please!
@@ -161,21 +163,19 @@ class TaskJuggler
         rText = ''
 
         taskList.each do |task|
-          rText += "== <nowiki>[#{task.query_alert(query)}] Task: " +
-            "#{task.name}</nowiki> (#{task.fullId}) ==\n\n"
+          rText += "=== <nowiki>[#{task.query_alert(query)}] Task: " +
+            "#{task.name}</nowiki> (#{task.fullId}) ===\n\n"
           rText += task.query_journalmessages(query).richText.inputText + "\n\n"
         end
       end
 
       # Now convert the RichText markup String into RichTextIntermediate
       # format.
-      begin
-        rti = RichText.new(rText, RTFHandlers.create(@project)).
-          generateIntermediateFormat
-      rescue RichTextException => msg
-        $stderr.puts "Error while processing Rich Text\n" +
-                     "Line #{msg.lineNo}: #{msg.text}\n" +
-                     "#{msg.line}"
+      unless (rti = RichText.new(rText, RTFHandlers.create(@project),
+                                 @project.messageHandler).
+                                 generateIntermediateFormat)
+        @project.messageHandler.warning(
+          'res_dashboard', 'Syntax error in dashboard text')
         return nil
       end
       # No section numbers, please!

data/lib/ResourceScenario.rb

@@ -193,7 +193,7 @@ class TaskJuggler
                 "Resource #{@property.fullId} has multiple conflicting " +
                 "bookings for #{@scoreboard.idxToDate(sbIdx)}. The " +
                 "conflicting tasks are #{@scoreboard[sbIdx].fullId} and " +
-                "#{booking.task.fullId}.", true, booking.sourceFileInfo)
+                "#{booking.task.fullId}.", booking.sourceFileInfo)
         end
         val = @scoreboard[sbIdx]
         if ((val & 2) != 0 && booking.overtime < 1)
@@ -202,7 +202,7 @@ class TaskJuggler
           if booking.sloppy < 1
             error('booking_no_duty',
                   "Resource #{@property.fullId} has no duty at " +
-                  "#{@scoreboard.idxToDate(sbIdx)}.", true,
+                  "#{@scoreboard.idxToDate(sbIdx)}.",
                   booking.sourceFileInfo)
           end
           return false
@@ -213,7 +213,7 @@ class TaskJuggler
           if booking.sloppy < 2
             error('booking_on_vacation',
                   "Resource #{@property.fullId} is on vacation at " +
-                  "#{@scoreboard.idxToDate(sbIdx)}.", true,
+                  "#{@scoreboard.idxToDate(sbIdx)}.",
                   booking.sourceFileInfo)
           end
           return false

data/lib/RichText.rb

@@ -12,6 +12,7 @@
 
 require 'RichTextElement'
 require 'RichTextParser'
+require 'MessageHandler'
 
 class TaskJuggler
 
@@ -70,17 +71,18 @@ class TaskJuggler
   #
   class RichText
 
-    attr_reader :inputText
+    attr_reader :inputText, :messageHandler
 
     # Create a rich text object by passing a String with markup elements to it.
     # _text_ must be plain text with MediaWiki compatible markup elements. In
     # case an error occurs, an exception of type TjException will be raised.
     # _functionHandlers_ is a Hash that maps RichTextFunctionHandler objects
     # by their function name.
-    def initialize(text, functionHandlers = [])
+    def initialize(text, functionHandlers = [], messageHandler = nil)
       # Keep a copy of the original text.
       @inputText = text
       @functionHandlers = functionHandlers
+      @messageHandler = messageHandler || MessageHandler.new
     end
 
     # Convert the @inputText into an abstract syntax tree that can then be
@@ -93,10 +95,12 @@ class TaskJuggler
         rti.registerFunctionHandler(h)
       end
       # Parse the input text into an abstract syntax tree.
-      parser = RichTextParser.new(rti, sectionCounter, tokenSet)
+      parser = RichTextParser.new(@messageHandler, rti, sectionCounter,
+                                  tokenSet)
       parser.open(@inputText)
       # Parse the input text and convert it to the intermediate representation.
-      tree = parser.parse('richtext')
+      return nil if (tree = parser.parse('richtext')) == false
+
       # In case the result is empty, use an empty RichTextElement as result
       tree = RichTextElement.new(rti, :richtext, nil) unless tree
       tree.cleanUp
@@ -124,7 +128,7 @@ class TaskJuggler
   # generated by RichText::generateIntermediateFormat.
   class RichTextIntermediate
 
-    attr_reader :richText
+    attr_reader :richText, :functionHandlers
     attr_accessor :blockMode, :sectionNumbers,
                   :lineWidth, :indent, :titleIndent, :parIndent, :listIndent,
                   :preIndent,
@@ -197,12 +201,16 @@ class TaskJuggler
 
     # Convert the intermediate format into a plain text String object.
     def to_s
-      @tree.to_s
+      str = @tree.to_s
+      str.chomp! while str[-1] == ?\n
+      str
     end
 
     # Convert the intermediate format into a XMLElement objects tree.
     def to_html
-      @tree.to_html
+      html = @tree.to_html
+      html.chomp! while html[-1] == ?\n
+      html
     end
 
     # Convert the intermediate format into a tagged syntax String object.

data/lib/RichTextDocument.rb

@@ -34,6 +34,7 @@ class TaskJuggler
       @sectionCounter = [ 0, 0, 0 ]
       @linkTarget = nil
       @toc = nil
+      @anchors = []
     end
 
     # Register a new RichTextFunctionHandler for this document.
@@ -53,17 +54,20 @@ class TaskJuggler
     # Call this method to generate a table of contents for all files that were
     # registered so far. The table of contents is stored internally and will be
     # used when the document is produced in a new format. This function also
-    # collects a list of all snip names to @snipNames and gathers a list of
+    # collects a list of all snip names to @anchors and gathers a list of
     # all references to other snippets in @references. As these two lists will
     # be used by RichTextDocument#checkInternalReferences this function must be
     # called first.
     def tableOfContents
       @toc = TableOfContents.new
       @references = {}
-      @snipNames = []
+      @anchors = []
       @snippets.each do |snip|
         snip.tableOfContents(@toc, snip.name)
-        @snipNames << snip.name
+        @anchors << snip.name
+        @toc.each do |tocEntry|
+          @anchors << snip.name + '#' + tocEntry.tag
+        end
         (refs = snip.internalReferences).empty? ||
           @references[snip.name] = refs
       end
@@ -73,7 +77,7 @@ class TaskJuggler
     def checkInternalReferences
       @references.each do |snip, refs|
         refs.each do |reference|
-          unless @snipNames.include?(reference)
+          unless @anchors.include?(reference)
             # TODO: Probably an Exception is cleaner here.
             puts "Warning: Rich text file #{snip} references unknown " +
                  "object #{reference}"

data/lib/RichTextElement.rb

@@ -216,7 +216,6 @@ class TaskJuggler
       when :href
       when :blockfunc
       when :inlinefunc
-        noChilds = true
         checkHandler
         pre = @richText.functionHandler(@data[0]).to_s(@data[1])
       when :italic
@@ -224,7 +223,7 @@ class TaskJuggler
       when :code
       when :text
       else
-        raise TjException.new, "Unknown RichTextElement category #{@category}"
+        raise "Unknown RichTextElement category #{@category}"
       end
 
       pre + children_to_s + post
@@ -323,7 +322,7 @@ class TaskJuggler
         pre = "<blockfunc:#{@data[0]}"
         if @data[1]
           @data[1].keys.sort.each do |key|
-            pre += " #{key}=\"#{@data[1][key]}\""
+            pre += " #{key}=\"#{@data[1][key].gsub(/"/, '\"')}\""
           end
         end
         post = "/>"
@@ -348,7 +347,7 @@ class TaskJuggler
         pre = '['
         post = ']'
       else
-        raise TjException.new, "Unknown RichTextElement category #{@category}"
+        raise "Unknown RichTextElement category #{@category}"
       end
 
       out = ''
@@ -425,14 +424,17 @@ class TaskJuggler
       when :numberitem4
         XMLElement.new('li')
       when :img
-        el = XMLElement.new('img', 'src' => @data.fileName)
-        el['alt'] = @data.altText if @data.altText
-        if @data.verticalAlign
-          el['style'] = "vertical-align:#{@data.verticalAlign}; "
-        end
-        el
+        htmlObject
       when :ref
-        XMLElement.new('a', 'href' => "#{@data}.html")
+        href = if @data.include?('#')
+                 # If the @data includes a reference to an anchor, we put the
+                 # anchor part after the .html extension.
+                 pre, post = @data.split('#')
+                 pre + '.html#' + post
+               else
+                 @data + '.html'
+               end
+        XMLElement.new('a', 'href' => href)
       when :href
         a = XMLElement.new('a', 'href' => @data.to_s)
         a['target'] = @richText.linkTarget if @richText.linkTarget
@@ -455,7 +457,7 @@ class TaskJuggler
         noChilds = true
         XMLText.new(@children[0])
       else
-        raise TjException.new, "Unknown RichTextElement category #{@category}"
+        raise "Unknown RichTextElement category #{@category}"
       end
 
       # Some elements never have leaves.
@@ -530,6 +532,27 @@ class TaskJuggler
       el
     end
 
+    def htmlObject
+      fileTypes = { 'png' => { 'type' => 'image/png' },
+                    'gif' => { 'type' => 'image/gif' },
+                    'jpg' => { 'type' => 'image/jpg' },
+                    'svg' => { 'type' => 'image/svg+xml', 'class' => 'img' }}
+      # Error checking must have been done in the parser!
+      # File types must be in sync with
+      # RichTextSyntaxRules::rule_plainTextWithLinks
+      return nil unless (index = @data.fileName.rindex('.'))
+      extension = @data.fileName[index + 1..-1].downcase
+      return nil unless (attributes = fileTypes[extension])
+      attributes['data'] = @data.fileName
+
+      el = XMLElement.new('object', attributes)
+      el['alt'] = @data.altText if @data.altText
+      if @data.verticalAlign
+        el['style'] = "vertical-align:#{@data.verticalAlign}; "
+      end
+      el
+    end
+
     def textBlockFormat(indent, label, str, width)
       labLen = label.length
       TextFormatter.new(width, indent + labLen, indent).format(label + str)

data/lib/RichTextFunctionHandler.rb

@@ -12,29 +12,28 @@
 
 class TaskJuggler
 
-  # This class is the abstract base class for all RichText function handlers. A
-  # function handler is responsible for a certain function such as 'example' or
-  # 'query'. functions are used in internal RichText references such as
+  # This class is the abstract base class for all RichText function handlers.
+  # A function handler is responsible for a certain function such as 'example'
+  # or 'query'. functions are used in internal RichText references such as
   # '[[example:Allocation 2]]'. 'example' is the function, 'Allocation' is the
-  # path and '2' is the first argument. Arguments are optional. function handler
-  # can turn such internal references into Strings or XMLElement trees.
-  # Therefor, each derived handler needs to implement a to_s, to_html and
-  # to_tagged method that takes two parameter. The first is the path, the second
-  # is the argument Array.
+  # path and '2' is the first argument. Arguments are optional. The function
+  # handler can turn such internal references into Strings or XMLElement
+  # trees. Therefor, each derived handler needs to implement a to_s, to_html
+  # and to_tagged method that takes two parameter. The first is the path, the
+  # second is the argument Array.
   class RichTextFunctionHandler
 
     attr_reader :function, :blockFunction
 
-    def initialize(project, function, sourceFileInfo = nil)
-      @project = project
+    def initialize(messageHandler, function, sourceFileInfo = nil)
+      @messageHandler = messageHandler
       @function = function
       @blockFunction = false
       @sourceFileInfo = sourceFileInfo
     end
 
     def error(id, text)
-      message = Message.new(id, 'error', text, nil, nil, @sourceFileInfo)
-      @project.sendMessage(message)
+      @messageHandler.error(id, text, @sourceFileInfo) if @messageHandler
     end
 
   end

data/lib/RichTextParser.rb

@@ -27,8 +27,9 @@ class TaskJuggler
 
     # Create the parser and initialize the rule set. _rt_ is the RichText object
     # the resulting tree of RichTextElement objects should belong to.
-    def initialize(rti, sectionCounter = [ 0, 0, 0, 0 ], tokenSet = nil)
-      super()
+    def initialize(messageHandler, rti, sectionCounter = [ 0, 0, 0, 0 ],
+                   tokenSet = nil)
+      super(messageHandler)
       @richTextI = rti
       # These are the tokens that can be returned by the RichTextScanner.
       @variables = %w( LINEBREAK SPACE WORD BOLD ITALIC CODE BOLDITALIC PRE
@@ -53,7 +54,8 @@ class TaskJuggler
     def open(text)
       # Make sure that the last line is properly terminated with a newline.
       # Multiple newlines at the end are simply ignored.
-      @scanner = RichTextScanner.new(text + "\n\n")
+      @scanner = RichTextScanner.new(text + "\n\n", @messageHandler)
+      @scanner.open(true)
     end
 
     # Get the next token from the scanner.

data/lib/RichTextScanner.rb

@@ -11,623 +11,202 @@
 #
 
 require 'UTF8String'
+require 'TextScanner'
 
 class TaskJuggler
 
   # The RichTextScanner is used by the RichTextParser to chop the input text
-  # into digestable tokens. The parser and the scanner only communicate over
-  # RichTextScanner#nextToken and RichTextScanner#returnToken. The scanner can
-  # break the text into words and special tokens.
-  class RichTextScanner
-
-    # Create the RichTextScanner object and initialize all state variables.
-    def initialize(text)
-      # The token buffer is used to hold a returned token. Only one token can
-      # be returned at a time.
-      @tokenBuffer = nil
-      # A reference to the input text.
-      @text = text
-      # The reference text should not change during processing. So we can
-      # determine the length upfront. It's frequently used.
-      @textLength = text.length
-      # The number of current line.
-      @lineNo = 1
-      # This is the current position withing @text.
-      @pos = 0
-      # This flag is set to true whenever we are at the start of a text line.
-      @beginOfLine = true
-      # This is the position of the start of the currently processed line.
-      # It's only used for error reporting.
-      @lineStart = 0
-      # This variable stores the mode that the parser is operating in. The
-      # following modes are supported:
-      # :wiki : accept supported MediaWiki subset plus TJ extensions
-      # :nowiki : ignore most markup except for the </nowiki> token
-      # :funcarg : parse name and parameters of an block  or inline parser
-      # function.
-      @mode = :wiki
-      # Enable to trigger printout instead of exception.
-      @debug = false
-    end
-
-    # This is a wrapper for nextToken only used for debugging.
-    #def nextToken
-    #  tok = nextTokenI
-    #  raise "Token Error:" unless tok && tok[0] && tok[1]
-    #  puts "#{tok[0]}: #{tok[1]}"
-    #  tok
-    #end
-
-    # Return the next token from the input text.
-    def nextToken
-      # If we have a returned token, this is returned first.
-      if @tokenBuffer
-        tok = @tokenBuffer
-        @tokenBuffer = nil
-        return tok
-      end
-
-      if @mode == :funcarg
-        return nextTokenFuncArg
-      elsif @mode == :href
-        return nextTokenHRef
-      elsif @mode == :ref
-        return nextTokenRef
-      end
-      if @beginOfLine && @mode == :wiki
-        if (res = nextTokenWikiBOL)
-          return res
-        end
-      end
-
-      # Many inline control character sequences consit of multiple characters.
-      # In case of incomplete sequences, we roll back to the start character
-      # and set the ignoreInlineMarkup flag to simply treat them as normal
-      # text.
-      @ignoreInlineMarkup = false
-      loop do
-        if res = (@mode == :wiki ? nextTokenWikiInline : nextTokenNoWikiInline)
-          return res
-        end
-      end
-    end
-
-    # Return the last issued token to the token buffer.
-    def returnToken(token)
-      unless @tokenBuffer.nil?
-        raise TjException.new, 'Token buffer overflow!'
-      end
-      @tokenBuffer = token
-    end
+  # into digestable tokens. It specializes the TextScanner class for RichText
+  # syntax. The scanner can operate in various modes. The current mode is
+  # context dependent. The following modes are supported:
+  #
+  # :bop :     at the begining of a paragraph.
+  # :bol :     at the begining of a line.
+  # :inline :  in the middle of a line
+  # :nowiki :  ignoring all MediaWiki special tokens
+  # :ref :     inside of a REF [[ .. ]]
+  # :href :    inside of an HREF [ .. ]
+  # :func :    inside of a block <[ .. ]> or inline <- .. -> function
+  class RichTextScanner < TextScanner
+
+    def initialize(masterFile, messageHandler)
+      tokenPatterns = [
+        # :bol mode rules
+        [ 'LINEBREAK', /\s*\n/, :bol, method('linebreak') ],
+        [ nil, /\s+/, :bol, method('inlineMode') ],
+
+        # :bop mode rules
+        [ 'PRE', / [^\n]+\n?/, :bop, method('pre') ],
+        [ nil, /\s*\n/, :bop, method('linebreak') ],
+
+        # :inline mode rules
+        [ 'SPACE', /[ \t\n]+/, :inline, method('space') ],
+
+        # :bop and :bol mode rules
+        [ 'INLINEFUNCSTART', /<-/, [ :bop, :bol, :inline ],
+          method('functionStart') ],
+        [ 'BLOCKFUNCSTART', /<\[/, [ :bop, :bol ], method('functionStart') ],
+        [ 'TITLE*', /={2,5}/, [ :bop, :bol ], method('titleStart') ],
+        [ 'TITLE*END', /={2,5}/, :inline, method('titleEnd') ],
+        [ 'BULLET*', /\*{1,4} /, [ :bop, :bol ], method('bullet') ],
+        [ 'NUMBER*', /\#{1,4} /, [ :bop, :bol ], method('number') ],
+        [ 'HLINE', /----/, [ :bop, :bol ], method('inlineMode') ],
+
+        # :bop, :bol and :inline mode rules
+        # The <nowiki> token puts the scanner into :nowiki mode.
+        [ nil, /<nowiki>/, [ :bop, :bol, :inline ], method('nowikiStart') ],
+        [ 'QUOTES', /'{2,5}/, [ :bop, :bol, :inline ], method('quotes') ],
+        [ 'REF', /\[\[/, [ :bop, :bol, :inline ], method('refStart') ],
+        [ 'HREF', /\[/, [ :bop, :bol, :inline], method('hrefStart') ],
+        [ 'WORD', /.[^ \n\t\[<']*/, [ :bop, :bol, :inline ],
+          method('inlineMode') ],
 
-    # Report the current cursor position.
-    def sourceFileInfo
-      [ @lineNo, @pos ]
-    end
+        # :nowiki mode rules
+        [ nil, /<\/nowiki>/, :nowiki, method('nowikiEnd') ],
+        [ 'WORD', /(<(?!\/nowiki>)|[^ \t\n<])+/, :nowiki ],
+        [ 'SPACE', /[ \t]+/, :nowiki ],
+        [ 'LINEBREAK', /\s*\n/, :nowiki ],
 
-    # This function makes more sense for parsers that process actual files. As
-    # we don't have a file name, we just return 'input text'.
-    def fileName
-      'input text'
-    end
+        # :ref mode rules
+        [ 'REFEND', /\]\]/, :ref, method('refEnd') ],
+        [ 'WORD', /(<(?!-)|(\](?!\])|[^|<\]]))+/, :ref ],
+        [ 'QUERY', /<-\w+->/, :ref, method('query') ],
+        [ 'LITERAL', /./, :ref ],
 
-    # The parser uses this function to report any errors during parsing.
-    def error(id, text, foo = nil, bar = nil)
-      if @debug
-        $stderr.puts "Line #{@lineNo}: #{text}\n" +
-                     "#{@text[@lineStart, @pos - @lineStart]}"
-      else
-        raise RichTextException.new(id, @lineNo, text,
-                                    @text[@lineStart, @pos - @lineStart])
-      end
-    end
+        # :href mode rules
+        [ 'HREFEND', /\]/, :href, method('hrefEnd') ],
+        [ 'WORD', /(<(?!-)|[^ \t\n\]<])+/, :href ],
+        [ 'QUERY', /<-\w+->/, :href, method('query') ],
+        [ 'SPACE', /[ \t\n]+/, :href ],
 
-  private
-
-    # Function arguments have the following formats:
-    #  <[blockfunc par1="value1" par2='value2']>
-    #  <-inlinefunc par1="value1" ... ->
-    def nextTokenFuncArg
-      token = [ '.', '<END>' ]
-      while (c = nextChar)
-        case c
-        when ' ', "\n", "\t"
-          if (tok = readBlanks(c))
-            token = tok
-            break
-          end
-        when '='
-          return [ '_=', '=' ]
-        when "'"
-          return readString(c)
-        when '"'
-          return readString(c)
-        when 'a'..'z', 'A'..'Z', '_'
-          return readId(c)
-        when ']'
-          if nextChar == '>'
-            @mode = :wiki
-            return [ 'BLOCKFUNCEND', ']>' ]
-          end
-          returnChar
-        when '-'
-          if nextChar == '>'
-            @mode = :wiki
-            return [ 'INLINEFUNCEND', '->' ]
-          end
-          returnChar
-        end
-      end
-      token
+        # :func mode rules
+        [ 'INLINEFUNCEND', /->/ , :func, method('functionEnd') ],
+        [ 'BLOCKFUNCEND', /\]>/, :func, method('functionEnd') ],
+        [ 'ID', /[a-zA-Z_]\w*/, :func ],
+        [ 'STRING', /"(\\"|[^"])*"/, :func, method('dqString') ],
+        [ 'STRING', /'(\\'|[^'])*'/, :func, method('sqString') ],
+        [ nil, /[ \t\n]+/, :func ],
+        [ 'LITERAL', /./, :func ]
+      ]
+      super(masterFile, messageHandler, tokenPatterns, :bop)
     end
 
-    def nextTokenRef
-      c = nextChar
-      return [ '.', '<END' ] if c.nil?
+    private
 
-      return [ 'LITERAL', '|' ] if c == '|'
-
-      if c == ']' && peek == ']'
-        nextChar
-        @mode = :wiki
-        return [ 'REFEND', ']]' ]
-      end
-
-      token = c
-      while (c = nextChar)
-        break if c.nil?
-        if c == '|' || (c == ']' && peek == ']')
-          returnChar
-          break
-        end
-        token << c
+    def space(type, match)
+      if match.index("\n")
+        # If the match contains a linebreak we switch to :bol mode.
+        self.mode = :bol
+        # And return an empty string.
+        match = ''
       end
-      [ 'WORD', token ]
+      [type, match ]
     end
 
-    def nextTokenHRef
-      token = [ '.', '<END>' ]
-      while (c = nextChar)
-        if c.nil?
-          # We've reached the end of the text.
-          return token
-        elsif c == ' ' || c == "\t" || c == "\n"
-          # Sequences of tabs, spaces and newlines are treated as token
-          # boundaries, but otherwise they are ignored.
-          readSequence(" \n\t")
-          return [ 'SPACE', ' ' ]
-        elsif c == '<' && !@ignoreInlineMarkup
-          if nextChar == '-' && isIdStart(peek(1))
-            token = readId('', 'QUERY')
-            unless nextChar == '-' && nextChar == '>'
-              error('unterminated_query',
-                    "Inline query must be terminated with '->'")
-            end
-            return token
-          else
-            # It's not a query.
-            returnChar(2)
-            @ignoreInlineMarkup = true
-            next
-          end
-        elsif c == ']'
-          @mode = :wiki
-          return [ 'HREFEND', ']' ]
-        else
-          return nextTokenWord(c)
-        end
-      end
-      token
+    def linebreak(type, match)
+      self.mode = :bop
+      [ type, match ]
     end
 
-    def nextTokenWikiBOL
-      # Some characters have only a special meaning at the start of the line.
-      # When the last token pushed the cursor into a new line, this flag is set
-      # to true.
-
-      # Reset the flag again.
-      @beginOfLine = false
-
-      # We already know that the last newline was a real linebreak. Further
-      # newlines can safely be ignored.
-      readSequence("\n")
-
-      # All the lead characters of a token here also need to be registered
-      # with nextTokenNewline!
-      case (c = nextChar)
-      when '='
-        # Headings start with 2 or more = and must be followed by a space.
-        level = readSequenceMax('=', 5)
-        if level == 1
-          # 1 = does not mean anything. Push it back and process it as normal
-          # text further down.
-          returnChar
-        else
-          # Between the = characters and the title text must be exactly one
-          # space.
-          return [ "TITLE#{level - 1}", '=' * level ] if nextChar == ' '
-          # If that's missing, The = are treated as normal text further down.
-          returnChar(level + 1)
-        end
-      when '-'
-        # Horizontal ruler. Must have exactly 4 -.
-        level = readSequenceMax('-', 4)
-        return [ "HLINE", '-' * 4 ] if level == 4
-        returnChar(level)
-      when '*'
-        # Bullet lists start with one to three * characters.
-        level = readSequenceMax('*', 4)
-        # Between the * characters and the bullet text must be exactly one
-        # space.
-        return [ "BULLET#{level}", '*' * level ] if nextChar == ' '
-        # If that's missing, The # are treated as normal text further down.
-        returnChar(level + 1)
-      when '#'
-        # Numbered list start with one to three # characters.
-        level = readSequenceMax('#', 4)
-        # Between the # characters and the bullet text must be exactly one
-        # space.
-        return [ "NUMBER#{level}", '#' * level ] if nextChar == ' '
-        # If that's missing, The # are treated as normal text further down.
-        returnChar(level + 1)
-      when '<'
-        # This may be the start of a block generating function.
-        if nextChar == '['
-          # Switch the parser to block function argument parsing mode.
-          @mode = :funcarg
-          return [ 'BLOCKFUNCSTART', '<[' ]
-        end
-        # Maybe not.
-        returnChar(2)
-      when ' '
-        # Lines that start with a space are treated as verbatim text.
-        return [ "PRE", readCode ] if (c = peek) && c != "\n"
-      else
-        # If the character is not a known control character we push it back
-        # and treat it as normal text further down.
-        returnChar
-      end
-
-      return nil
+    def inlineMode(type, match)
+      self.mode = :inline
+      [ type, match ]
     end
 
-    def nextTokenWikiInline
-      c = nextChar
-      if c.nil?
-        # We've reached the end of the text.
-        [ '.', '<END>' ]
-      elsif c == ' ' || c == "\t"
-        # Sequences of tabs or spaces are treated as token boundaries, but
-        # otherwise they are ignored.
-        readSequence(" \t")
-        [ 'SPACE', ' ' ]
-      elsif c == "'" && !@ignoreInlineMarkup
-        # Sequence of 2 ' means italic, 3 ' means bold, 4 ' means monospaced
-        # code, 5 ' means italic and bold. Anything else is just normal text.
-        level = readSequenceMax("'", 5)
-        if level == 2
-          [ 'ITALIC', "'" * level ]
-        elsif level == 3
-          [ 'BOLD', "'" * level ]
-        elsif level == 4
-          [ 'CODE', "'" * level ]
-        elsif level == 5
-          [ 'BOLDITALIC', "'" * level ]
-        else
-          # We have not found the right syntax. Treat the found characters as
-          # normal text.  Push all ' back and start again but ignoring the '
-          # code for once.
-          returnChar(level)
-          @ignoreInlineMarkup = true
-          nil
-        end
-      elsif c == '=' && !@ignoreInlineMarkup
-        level = readSequenceMax('=', 5)
-        if level > 1
-          [ "TITLE#{level - 1}END", '=' * level ]
-        else
-          # We have not found the right syntax. Treat found characters as
-          # normal text.  Push all = back and start again but ignoring the =
-          # code for once.
-          returnChar(level)
-          @ignoreInlineMarkup = true
-          nil
-        end
-      elsif c == '['
-        level = readSequenceMax('[', 2)
-        if level == 1
-          @mode = :href
-          [ 'HREF' , '[' ]
-        else
-          @mode = :ref
-          [ 'REF', '[[' ]
-        end
-      elsif c == ']' && peek == ']'
-        nextChar
-        [ 'REFEND', ']]' ]
-      elsif c == "\n"
-        nextTokenNewline
-      elsif c == '<' && !@ignoreInlineMarkup
-        nextTokenOpenAngle
-      else
-        nextTokenWord(c)
-      end
+    def titleStart(type, match)
+      self.mode = :inline
+      [ "TITLE#{match.length - 1}", match ]
     end
 
-    def nextTokenNoWikiInline
-      c = nextChar
-      if c.nil?
-        # We've reached the end of the text.
-        [ '.', '<END>' ]
-      elsif c == ' ' || c == "\t"
-        # Sequences of tabs or spaces are treated as token boundaries, but
-        # otherwise they are ignored.
-        readSequence(" \t")
-        [ 'SPACE', ' ' ]
-      elsif c == "\n"
-        nextTokenNewline
-      elsif c == '<' && !@ignoreInlineMarkup
-        nextTokenOpenAngle
-      else
-        nextTokenWord(c)
-      end
+    def titleEnd(type, match)
+      [ "TITLE#{match.length - 1}END", match ]
     end
 
-    # We've just read a newline. Now we need to figure out whether this is a
-    # LINEBREAK or just a SPACE. This is determined by looking at the next
-    # character.
-    def nextTokenNewline
-      # Newlines are pretty important as they can terminate blocks and turn
-      # the next character into the start of a control sequence.
-      # Hard linebreaks consist of a newline followed by another newline or
-      # any of the begin-of-line control characters.
-      if (c = nextChar).nil?
-        # We hit the end of the text.
-        [ '.', '<END>' ]
-      elsif c == '<' && peekMatch('[')
-        # the '<' can be a start of a block (BLOCKFUNCSTART) or inline text
-        # (INLINEFUNCSTART). Only for the first case the linebreak is real.
-        returnChar if c != "\n"
-        # The next character may be a control character.
-        @beginOfLine = true
-        [ 'LINEBREAK', "\n" ]
-      elsif "\n*#=-".include?(c)
-        # These characters correspond to the first characters of a block
-        # element. When they are found at the begin of the line, the newline
-        # was really a line break.
-        returnChar if c != "\n"
-        # The next character may be a control character.
-        @beginOfLine = true
-        [ 'LINEBREAK', "\n" ]
-      else
-        # Single line breaks are treated as spaces. Return the char after
-        # the newline and start with this one again.
-        returnChar
-        [ 'SPACE', ' ' ]
-      end
+    def bullet(type, match)
+      self.mode = :inline
+      [ "BULLET#{match.length - 1}", match ]
     end
 
-    def nextTokenOpenAngle
-      if peekMatch('nowiki>')
-        # Turn most wiki markup interpretation off.
-        @pos += 'nowiki>'.length
-        @mode = :nowiki
-      elsif peekMatch('/nowiki>')
-        # Turn most wiki markup interpretation on.
-        @pos += '/nowiki>'.length
-        @mode = :wiki
-      elsif peekMatch('-') && @mode == :wiki
-        nextChar
-        # Switch the parser to function argument parsing mode.
-        @mode = :funcarg
-        return [ 'INLINEFUNCSTART', '<-' ]
-      else
-        # We've not found a valid control sequence. Push back the character
-        # and make sure we treat it as a normal character.
-        @ignoreInlineMarkup = true
-        returnChar
-      end
-      nil
+    def number(type, match)
+      self.mode = :inline
+      [ "NUMBER#{match.length - 1}", match ]
     end
 
-    # _c_ does not match any start of a control sequence, so we read
-    # characters until we find the end of the word.
-    def nextTokenWord(c)
-      # Reset this flag again.
-      @ignoreInlineMarkup = false
-      str = ''
-      str << c
-      # Now we can collect characters of a word until we hit a whitespace.
-      while (c = nextChar) && !" \n\t".include?(c)
-        case @mode
-        when :wiki
-          # Or at least two ' characters in a row.
-          break if c == "'" && peek == "'"
-          # Or a ] or <
-          break if ']<'.include?(c)
-        when :href
-          # Look for - of the end mark -> end ']'
-          break if '-]<'.include?(c)
-        else
-          # Make sure we find the </nowiki> tag even within a word.
-          break if c == '<'
-        end
-        str << c
-      end
-      # Return the character that indicated the word end.
-      returnChar
-      [ 'WORD', str ]
+    def quotes(type, match)
+      self.mode = :inline
+      types = [ nil, nil, 'ITALIC', 'BOLD' , 'CODE', 'BOLDITALIC' ]
+      [ types[match.length], match ]
     end
 
-    # Deliver the next character. Keep track of the cursor position. In case we
-    # reach the end, nil is returned.
-    def nextChar
-      if @pos >= @textLength
-        # Correct @pos so that returnChar works properly but mutliple reads of
-        # EOT are ignored.
-        @pos = @textLength + 1
-        return nil
-      end
-      c = @text[@pos]
-      @pos += 1
-      if c == ?\n
-        @lineNo += 1
-        # Save the position of the line start for later use during error
-        # reporting. The line begins after the newline.
-        @lineStart = @pos
-      end
-      # Since Ruby 1.9 is returning Strings for String#[] we need to emulate
-      # this for Ruby 1.8.
-      '' << c
+    def nowikiStart(type, match)
+      self.mode = :nowiki
+      [ type, match ]
     end
 
-    # Return one or more characters. _n_ is the number of characters to move
-    # back the cursor.
-    def returnChar(n = 1)
-      crossedNewline = false
-      if @pos <= @textLength && @pos >= n
-        # Check for newlines and update @lineNo accordingly.
-        n.times do |i|
-          if @text[@pos - i - 1] == ?\n
-            crossedNewline = true
-            @lineNo -= 1
-          end
-        end
-        @pos -= n
-      end
-
-      # If we have crossed a newline during rewind, we have to find the start of
-      # the current line again.
-      if crossedNewline
-        @lineStart = @pos
-        @lineStart -= 1 while @lineStart > 0 && @text[@lineStart - 1] != ?\n
-      end
+    def nowikiEnd(type, match)
+      self.mode = :inline
+      [ type, match ]
     end
 
-    # Return a character further up the text without moving the cursor.
-    # _lookAhead_ is the number of characters to peek ahead. A value of 0 would
-    # return the last character provided by nextChar().
-    def peek(lookAhead = 1)
-      return nil if (@pos + lookAhead - 1) >= @textLength
-      # Since Ruby 1.9 is returning Strings for String#[] we need to emulate
-      # this for Ruby 1.8.
-      '' << @text[@pos + lookAhead - 1]
+    def functionStart(type, match)
+      # When restoring :bol or :bop mode, we need to switch to :inline mode.
+      @funcLastMode = (@scannerMode == :bop || @scannerMode == :bol) ?
+                      :inline : @scannerMode
+      self.mode = :func
+      [ type, match ]
     end
 
-    # Return true if the next characters match exactly the character sequence in
-    # word.
-    def peekMatch(word)
-      # Since Ruby 1.9 is returning Strings for String#[] we need to emulate
-      # this for Ruby 1.8.
-      ('' << @text[@pos, word.length]) == word
+    def functionEnd(type, match)
+      self.mode = @funcLastMode
+      @funcLastMode = nil
+      [ type, match ]
     end
 
-    # Read a sequence of characters that are all contained in the _chars_ Array.
-    # If a character is found that is not in _chars_ the method returns the so
-    # far found sequence of chars as String.
-    def readSequence(chars)
-      sequence = ''
-      while (c = nextChar) && chars.index(c)
-        sequence << c
-      end
-      # Push back the character that did no longer match.
-      returnChar
-      sequence
+    def pre(type, match)
+      [ type, match[1..-1] ]
     end
 
-    # Read a sequence of _c_ characters until a different character is found or
-    # _max_ count has been reached.
-    def readSequenceMax(c, max = 3)
-      i = 1
-      while nextChar == c && i < max
-        i += 1
-      end
-      # Return the non matching character.
-      returnChar
-      i
+    def dqString(type, match)
+      # Remove first and last character and remove backslashes from quoted
+      # double quotes.
+      [ type, match[1..-2].gsub(/\\"/, '"') ]
     end
 
-    # Read a block of pre-formatted text. All lines must start with a space
-    # character.
-    def readCode
-      tok = ''
-      loop do
-        # Read until the end of the line
-        while (c = nextChar) && c != "\n"
-          # Append a found characters.
-          tok << c
-        end
-        # Append the newline.
-        tok << c
-        # If the next line does not start with a space, we've reached the end of
-        # the code block.
-        if (c = nextChar) && c != ' '
-          break
-        end
-      end
-      returnChar
-      @beginOfLine = true
-      tok
+    def sqString(type, match)
+      # Remove first and last character and remove backslashes from quoted
+      # single quotes.
+      [ type, match[1..-2].gsub(/\\'/, "'") ]
     end
 
-    def readBlanks(c)
-      loop do
-        if c != ' ' && c != "\n" && c != "\t"
-          returnChar
-          return nil
-        end
-        c = nextChar
-      end
+    def query(type, match)
+      # Remove <- and ->.
+      [ type, match[2..-3] ]
     end
 
-    def isIdStart(c)
-      (('a'..'z') === c || ('A'..'Z') === c || c == '_')
+    def hrefStart(type, match)
+      # When restoring :bol or :bop mode, we need to switch to :inline mode.
+      @hrefLastMode = (@scannerMode == :bop || @scannerMode == :bol) ?
+                      :inline : @scannerMode
+      self.mode = :href
+      [ type, match ]
     end
 
-    def readId(c, tokenType = 'ID')
-      token = ""
-      token << c
-      while (c = nextChar) &&
-            (('a'..'z') === c || ('A'..'Z') === c || ('0'..'9')  === c ||
-             c == '_')
-        token << c
-      end
-      returnChar
-      return [ tokenType, token ]
+    def hrefEnd(type, match)
+      self.mode = @hrefLastMode
+      @hrefLastMode = nil
+      [ type, match ]
     end
 
-    def readString(terminator)
-      token = ""
-      while (c = nextChar) && c != terminator
-        if c == "\\"
-          # Terminators can be used as regular characters when prefixed by a \.
-          if (c = nextChar) && c != terminator
-            # \ followed by non-terminator. Just add both.
-            token << "\\"
-          end
-        end
-        token << c
-      end
-
-      [ 'STRING', token ]
+    def refStart(type, match)
+      self.mode = :ref
+      [ type, match ]
     end
-  end
-
-  # Exception raised by the RichTextScanner in case of processing errors. Its
-  # primary purpose is to carry the id, lineNo, error message and the currently
-  # parsed line information.
-  class RichTextException < RuntimeError
 
-    attr_reader :lineNo, :id, :text, :line
-
-    def initialize(id, lineNo, msgText, line)
-      @id = id
-      @lineNo = lineNo
-      @text = msgText
-      @line = line
+    def refEnd(type, match)
+      self.mode = :inline
+      [ type, match ]
     end
 
   end
 
 end
-