taskjuggler 0.0.7 → 0.0.8

data/lib/SyntaxReference.rb

@@ -196,17 +196,16 @@ class TaskJuggler
       # Push pattern onto 'stack'.
       stack[pattern] = true
 
-      if pattern[0] == '_{' && pattern[2] == '_}'
+      if pattern[0][1] == '{' && pattern[2][1] == '}'
         # We have found an optional attribute pattern!
         return attributes(pattern[1], false)
       end
 
       # If a token of the pattern is a reference, we recursively
       # follow the reference to the next pattern.
-      pattern.each do |token|
-        if token[0] == ?!
-          token = token.slice(1, token.length - 1)
-          rule = @parser.rules[token]
+      pattern.each do |type, name|
+        if type == :reference
+          rule = @parser.rules[name]
           # Rules with multiple patterns won't lead to attributes.
           next if rule.patterns.length > 1
 
@@ -222,23 +221,23 @@ class TaskJuggler
     # rule. The patterns are returned as a hash. For each pattern the hashed
     # boolean value specifies whether the attribute is scenario specific or not.
     def attributes(token, scenarioSpecific)
-      raise "Token #{token} must reference a rule" if token[0] != ?!
-      token = token.slice(1, token.length - 1)
+      raise "Token #{token} must reference a rule" if token[0] != :reference
+      token = token[1]
       # Find the matching rule.
       rule = @parser.rules[token]
       attrs = {}
       # Now we look at the first token of each pattern.
       rule.patterns.each do |pattern|
-        if pattern[0][0] == ?_
+        if pattern[0][0] == :literal
           # If it's a terminal symbol, we found what we are looking for. We add
           # it to the attrs hash and mark it as non scenario specific.
           attrs[pattern] = scenarioSpecific
-        elsif pattern[0] == '!scenarioIdCol'
+        elsif pattern[0][0] == :reference && pattern[0][1] == :scenarioIdCol
           # A reference to the !scenarioId rule marks the next token of the
           # pattern as a reference to a rule with all scenario specific
           # attributes.
           attrs.merge!(attributes(pattern[1], true))
-        elsif pattern[0][0] == ?!
+        elsif pattern[0][0] == :reference
           # In case we have a reference to another rule, we just follow the
           # reference. If the pattern is documented we don't have to follow the
           # reference. We can use the pattern instead.

data/lib/TaskJuggler.rb

@@ -14,6 +14,8 @@ require 'drb'
 require 'Project'
 require 'MessageHandler'
 require 'Log'
+# Only needed during profiling.
+#require 'ruby-prof'
 
 # The TaskJuggler class models the object that provides access to the
 # fundamental features of the TaskJuggler software. It can read project
@@ -41,6 +43,7 @@ class TaskJuggler
     master = true
     @project = nil
 
+    #RubyProf.start
     @parser = ProjectFileParser.new(@messageHandler)
     files.each do |file|
       begin
@@ -54,7 +57,7 @@ class TaskJuggler
       end
       if master
         # The first file is considered the master file.
-        if (@project = @parser.parse('project')) == false
+        if (@project = @parser.parse(:project)) == false
           Log.exit('parser')
           return false
         end
@@ -62,7 +65,7 @@ class TaskJuggler
       else
         # All other files.
         @parser.setGlobalMacros
-        if @parser.parse('properties') == false
+        if @parser.parse(:propertiesFile) == false
           Log.exit('parser')
           return false
         end
@@ -71,6 +74,16 @@ class TaskJuggler
       @parser.close
     end
 
+    #profile = RubyProf.stop
+    #printer = RubyProf::GraphHtmlPrinter.new(profile)
+    #File.open("profile.html", "w") do |file|
+    #  printer.print(file)
+    #end
+    #printer = RubyProf::CallTreePrinter.new(profile)
+    #File.open("profile.clt", "w") do |file|
+    #  printer.print(file)
+    #end
+
     # For the report server mode we may need to keep the parser. Otherwise,
     # destroy it.
     @parser = nil unless keepParser
@@ -123,12 +136,23 @@ class TaskJuggler
   # this method can be called. It returns true if no error occured, false
   # otherwise.
   def generateReports(outputDir = './')
+    @project.checkReports
     outputDir += '/' unless outputDir.empty? || outputDir[-1] == '/'
     @project.outputDir = outputDir
     Log.enter('reports', 'Generating reports ...')
 
     begin
+      #RubyProf.start
       @project.generateReports(@maxCpuCores)
+      #profile = RubyProf.stop
+      #printer = RubyProf::GraphHtmlPrinter.new(profile)
+      #File.open("profile.html", "w") do |file|
+      #  printer.print(file)
+      #end
+      #printer = RubyProf::CallTreePrinter.new(profile)
+      #File.open("profile.clt", "w") do |file|
+      #  printer.print(file)
+      #end
     rescue TjException => msg
       if msg.message && !msg.message.empty?
         @messageHandler.critical('generate_reports', msg.message)
@@ -186,7 +210,7 @@ class TaskJuggler
       # Make sure we don't use data from old time sheets or Journal entries.
       @project.timeSheets.clear
       @project['journal'] = Journal.new
-      return false unless (ts = parseFile(fileName, 'timeSheetFile'))
+      return false unless (ts = parseFile(fileName, :timeSheetFile))
       return false unless @project.checkTimeSheets
       queryAttrs = { 'project' => @project,
                      'property' => ts.resource,
@@ -215,7 +239,7 @@ class TaskJuggler
   def checkStatusSheet(fileName)
     begin
       Log.enter('checkStatusSheet', 'Parsing #{fileName} ...')
-      return false unless (ss = parseFile(fileName, 'statusSheetFile'))
+      return false unless (ss = parseFile(fileName, :statusSheetFile))
       queryAttrs = { 'project' => @project,
                      'property' => ss[0],
                      'scopeProperty' => nil,

data/lib/TaskScenario.rb

@@ -11,6 +11,7 @@
 #
 
 require 'ScenarioData'
+require 'DataCache'
 
 class TaskJuggler
 
@@ -24,6 +25,7 @@ class TaskJuggler
 
       # A list of all allocated leaf resources.
       @candidates = []
+      @dCache = DataCache.instance
     end
 
     # Call this function to reset all scheduling related data prior to
@@ -324,9 +326,19 @@ class TaskJuggler
     # be called to do some more housekeeping. It computes some derived data
     # based on the just scheduled values.
     def finishScheduling
-      # The completion degree calculation is done recursively. So we only call
-      # it for top-level tasks.
-      calcCompletion unless @property.parent
+      # Recursively descend into all child tasks.
+      @property.children.each do |task|
+        task.finishScheduling(@scenarioIdx)
+      end
+
+      if (parent = @property.parent)
+        # Add the assigned resources to the parent task list.
+        a('assignedresources').each do |resource|
+          unless parent['assignedresources', @scenarioIdx].include?(resource)
+            parent['assignedresources', @scenarioIdx] << resource
+          end
+        end
+      end
 
       # This list is no longer needed, so let's save some memory. Set it to
       # nil so we can detect accidental use.
@@ -450,7 +462,8 @@ class TaskJuggler
                   "#{dependency.gapDuration / (60 * 60 * 24)} days after " +
                   "#{dependency.onEnd ? 'end' : 'start'} of task " +
                   "#{task.fullId}. TaskJuggler cannot enforce this condition " +
-                  "because the task is scheduled ALAP (finish-to-start).")
+                  "because the task is scheduled ALAP (finish-to-start) or " +
+                  "has a fixed #{dependency.onEnd ? 'end' : 'start'} date.")
           end
         end
         if dependency.gapLength > 0
@@ -461,7 +474,8 @@ class TaskJuggler
                   "working days after " +
                   "#{dependency.onEnd ? 'end' : 'start'} of task " +
                   "#{task.fullId}. TaskJuggler cannot enforce this condition " +
-                  "because the task is scheduled ALAP (finish-to-start).")
+                  "because the task is scheduled ALAP (finish-to-start) or " +
+                  "has a fixed #{dependency.onEnd ? 'end' : 'start'} date.")
           end
         end
       end
@@ -483,7 +497,8 @@ class TaskJuggler
                   "#{dependency.gapDuration / (60 * 60 * 24)} days before " +
                   "#{dependency.onEnd ? 'end' : 'start'} of task " +
                   "#{task.fullId}. TaskJuggler cannot enforce this condition " +
-                  "because the task is scheduled ASAP (start-to-finish).")
+                  "because the task is scheduled ASAP (start-to-finish) or " +
+                  "has a fixed #{dependency.onEnd ? 'end' : 'start'} date.")
           end
         end
         if dependency.gapLength > 0
@@ -494,7 +509,8 @@ class TaskJuggler
                   "working days before " +
                   "#{dependency.onEnd ? 'end' : 'start'} of task " +
                   "#{task.fullId}. TaskJuggler cannot enforce this condition " +
-                  "because the task is scheduled ASAP (start-to-finish).")
+                  "because the task is scheduled ASAP (start-to-finish) or " +
+                  "has a fixed #{dependency.onEnd ? 'end' : 'start'} date.")
           end
         end
       end
@@ -543,7 +559,7 @@ class TaskJuggler
       # Check if we have been here before on this path.
       if path.include?([ @property, atEnd ])
         warning('loop_detected',
-                "Loop detected at #{atEnd ? 'end' : 'start'} " +
+                "Dependency loop detected at #{atEnd ? 'end' : 'start'} " +
                 "of task #{@property.fullId}", false)
         skip = true
         path.each do |t, e|
@@ -1145,8 +1161,15 @@ class TaskJuggler
     end
 
     def query_complete(query)
-      query.sortable = query.numerical = complete = a('complete').to_i
-      query.string = "#{complete}%"
+      # If we haven't calculated the value yet, calculate it first.
+      unless (complete = a('complete'))
+        calcCompletion
+        complete = a('complete')
+      end
+
+      query.sortable = query.numerical = complete
+      # For the string output, we only use integer numbers.
+      query.string = "#{complete.to_i}%"
     end
 
     # Compute the cost generated by this Task for a given Account during a given
@@ -1278,8 +1301,11 @@ class TaskJuggler
     # the report time frame.
     def query_resources(query)
       list = ''
+      return list unless @property.leaf?
+
       a('assignedresources').each do |resource|
-        if getAllocatedTime(query.startIdx, query.endIdx, resource) > 0.0
+        if resource.allocated?(@scenarioIdx,
+                               Interval.new(query.start, query.end), @property)
           list += ', ' unless list.empty?
           list += "#{resource.name} (#{resource.fullId})"
         end
@@ -1303,6 +1329,16 @@ class TaskJuggler
       end
     end
 
+    def query_status(query)
+      # If we haven't calculated the completion yet, calculate it first.
+      if (status = a('status')).empty?
+        calcCompletion
+        status = a('status')
+      end
+
+      query.string = status
+    end
+
     def query_targets(query)
       targetList = PropertyList.new(@project.tasks, false)
       targets(targetList, true)
@@ -1324,7 +1360,12 @@ class TaskJuggler
     # Compute the total time _resource_ or all resources are allocated during
     # interval specified by _startIdx_ and _endIdx_.
     def getAllocatedTime(startIdx, endIdx, resource = nil)
-      return 0.0 if a('milestone')
+      return 0.0 if a('milestone') || startIdx >= endIdx ||
+                    (resource && !a('assignedresources').include?(resource))
+
+      key = [ self, :TaskScenarioAllocatedTime, startIdx, endIdx, resource ].hash
+      allocatedTime = @dCache.load(key)
+      return allocatedTime if allocatedTime
 
       allocatedTime = 0.0
       if @property.container?
@@ -1344,14 +1385,19 @@ class TaskJuggler
           end
         end
       end
-      allocatedTime
+      @dCache.store(allocatedTime, key)
     end
 
     # Compute the effective work a _resource_ or all resources do during the
     # interval specified by _startIdx_ and _endIdx_. The effective work is the
     # actual work multiplied by the efficiency of the resource.
     def getEffectiveWork(startIdx, endIdx, resource = nil)
-      return 0.0 if a('milestone')
+      return 0.0 if a('milestone') || startIdx >= endIdx ||
+                    (resource && !a('assignedresources').include?(resource))
+
+      key = [ self, :TaskScenarioEffectiveWork, startIdx, endIdx, resource ].hash
+      workLoad = @dCache.load(key)
+      return workLoad if workLoad
 
       workLoad = 0.0
       if @property.container?
@@ -1370,7 +1416,7 @@ class TaskJuggler
           end
         end
       end
-      workLoad
+      @dCache.store(workLoad, key)
     end
 
     # Return a list of intervals that lay within _iv_ and are at least
@@ -1435,6 +1481,8 @@ class TaskJuggler
     # Returns true of the _resource_ is assigned to this task or any of its
     # children.
     def hasResourceAllocated?(interval, resource)
+      return false unless a('assignedresources').include?(resource)
+
       if @property.leaf?
         return resource.allocated?(@scenarioIdx, interval, @property)
       else
@@ -1701,6 +1749,7 @@ class TaskJuggler
     # specified Interval.
     def bookBookings
       scheduled = a('scheduled')
+      slotDuration = @project['scheduleGranularity']
       a('booking').each do |booking|
         unless booking.resource.leaf?
           error('booking_resource_not_leaf',
@@ -1711,7 +1760,6 @@ class TaskJuggler
           error('booking_forward_only',
                 "Only forward scheduled tasks may have booking statements.")
         end
-        slotDuration = @project['scheduleGranularity']
         booking.intervals.each do |interval|
           startIdx = @project.dateToIdx(interval.start)
           date = interval.start
@@ -1727,8 +1775,7 @@ class TaskJuggler
               # set to the begining of the first booked slot. The lastSlot
               # will be set to the last booked slot
               @lastSlot = date if @lastSlot.nil? || date > @lastSlot
-              @tentativeEnd = tEnd if @tentativeEnd.nil? ||
-                @tentativeEnd < tEnd
+              @tentativeEnd = tEnd if @tentativeEnd.nil? || @tentativeEnd < tEnd
               if !scheduled && (a('start').nil? || date < a('start'))
                 @property['start', @scenarioIdx] = date
               end
@@ -1737,7 +1784,7 @@ class TaskJuggler
                 @property['assignedresources', @scenarioIdx] << booking.resource
               end
             end
-            if a('length') > 0 && @project.isWorkingTime(date, tEnd)
+            if a('length') > 0 && @project.isWorkingTime(date)
               # For tasks with a 'length' we track the covered work time and
               # set the task to 'scheduled' when we have enough length.
               @doneLength += 1

data/lib/TextParser.rb

@@ -19,13 +19,11 @@ require 'Log'
 
 class TaskJuggler
 
-  # The TextParser implements a regular LALR parser. But it uses a recursive
-  # rule traversor instead of the more commonly found state machine generated by
-  # yacc-like tools. Since stack depths is not really an issue with a Ruby
-  # implementation this approach has one big advantage. The syntax of the parser
-  # can be modified during parsing. This allows support for languages that can
-  # extend themself. The TaskJuggler syntax is such an beast. Traditional yacc
-  # generated parsers would fail with such a syntax.
+  # The TextParser implements a somewhat modified LL(1) parser. It uses a
+  # dynamically compiled state machine. Dynamically means, that the syntax can
+  # be extended during the parse process. This allows support for languages
+  # that can extend their syntax during the parse process. The TaskJuggler
+  # syntax is such an beast.
   #
   # This class is just a base class. A complete parser would derive from this
   # class and implement the rule set and the functions _nextToken()_ and
@@ -39,12 +37,21 @@ class TaskJuggler
   # document a pattern, the functions TextParser#doc, TextParser#descr,
   # TextParser#also and TextParser#arg can be used.
   #
+  # In contrast to conventional LL grammars, we use a slightly improved syntax
+  # descriptions. Repeated patterns are not described by recursive call but we
+  # use a repeat flag for syntax rules that consists of repeatable patterns.
+  # This removes the need for recursion elimination when compiling the state
+  # machine and makes the syntax a lot more readable. However, it adds a bit
+  # more complexity to the state machine. Optional patterns are described by
+  # a rule flag, not by adding an empty pattern.
+  #
   # To start parsing the input the function TextParser#parse needs to be called
   # with the name of the start rule.
   class TextParser
 
     # Utility class so that we can distinguish Array results from the Array
-    # containing the results of a repeatable rule.
+    # containing the results of a repeatable rule. We define some merging
+    # method with a slightly different behaviour.
     class TextParserResultArray < Array
 
       def initialize
@@ -62,7 +69,6 @@ class TaskJuggler
           super
         end
       end
-
     end
 
     attr_reader :rules, :messageHandler
@@ -76,9 +82,15 @@ class TaskJuggler
       # Array to hold the token types that the scanner can return.
       @variables = []
       # An list of token types that are not allowed in the current context.
-      @badVariables = []
+      # For performance reasons we use a hash with the token as key. The value
+      # is irrelevant.
+      @blockedVariables = {}
       # The currently processed rule.
       @cr = nil
+
+      @states = {}
+      # The stack used by the FSM.
+      @stack = nil
     end
 
     # Limit the allowed tokens of the scanner to the subset passed by the
@@ -86,8 +98,13 @@ class TaskJuggler
     def limitTokenSet(tokenSet)
       return unless tokenSet
 
-      @badVariables = @variables.dup
-      @badVariables.delete_if { |v| tokenSet.include?(v) }
+      # Create a copy of all supported variables.
+      blockedVariables = @variables.dup
+      # Then delete all that are in the limited set.
+      blockedVariables.delete_if { |v| tokenSet.include?(v) }
+      # And convert the list into a Hash for faster lookups.
+      @blockedVariables = {}
+      blockedVariables.each { |v| @blockedVariables[v] = true }
     end
 
     # Call all methods that start with 'rule_' to initialize the rules.
@@ -108,6 +125,8 @@ class TaskJuggler
     # TextParser#repeatable will then implicitely operate on the most recently
     # added rule.
     def newRule(name)
+      # Use a symbol instead of a String.
+      name = name.intern
       raise "Fatal Error: Rule #{name} already exists" if @rules.has_key?(name)
 
       if block_given?
@@ -148,13 +167,26 @@ class TaskJuggler
     end
 
     # This function needs to be called whenever new rules or patterns have been
-    # added and before the next call to TextParser#parse.
+    # added and before the next call to TextParser#parse. It's perfectly ok to
+    # call this function from within a parse() call as long as the states that
+    # are currently on the stack have not been modified.
     def updateParserTables
-      @rules.each_value { |rule| rule.transitions = {} }
+      saveFsmStack
+      # Invalidate some cached data.
+      @rules.each_value { |rule| rule.flushCache }
+      @states = {}
+      # Generate the parser states for all patterns of all rules.
       @rules.each_value do |rule|
-        getTransitions(rule)
+        rule.generateStates.each do |s|
+          @states[[ s.rule, s.pattern, s.index ]] = s
+        end
         checkRule(rule)
       end
+      # Compute the transitions between the generated states.
+      @states.each_value do |state|
+        state.addTransitions(@states, @rules)
+      end
+      restoreFsmStack
     end
 
     # To parse the input this function needs to be called with the name of the
@@ -164,9 +196,8 @@ class TaskJuggler
     def parse(ruleName)
       @stack = []
       @@expectedTokens = []
-      updateParserTables
       begin
-        result = parseRuleR(@rules[ruleName])
+        result = parseFSM(@rules[ruleName])
       rescue TjException => msg
         if msg.message && !msg.message.empty?
           @messageHandler.critical('parse', msg.message)
@@ -181,17 +212,8 @@ class TaskJuggler
     # currently processed TextParser::Rule. Or return nil if we don't have a
     # current position.
     def sourceFileInfo
-      return nil if @stack.nil? || @stack.empty?
-      @stack.last.sourceFileInfo[0]
-    end
-
-    def matchingRules(keyword)
-      matches = []
-      @rules.each do |name, rule|
-        patIdx = rule.matchingPatternIndex('_' + keyword)
-        matches << [ rule, patIdx ] if patIdx
-      end
-      matches
+      return @scanner.sourceFileInfo if @stack.nil? || @stack.length <= 1
+      @stack.last.firstSourceFileInfo
     end
 
     def error(id, text, sfi = nil, data = nil)
@@ -218,79 +240,21 @@ class TaskJuggler
 
   private
 
-    # getTransitions recursively determines all possible target tokens
-    # that the _rule_ matches. A target token can either be a fixed token
-    # (prefixed with _), a variable token (prefixed with $) or an end token
-    # (just a .). The list of found target tokens is stored in the _transitions_
-    # list of the rule. For each rule pattern we store the transitions for this
-    # pattern in a token -> rule hash.
-    def getTransitions(rule)
-      # If we have processed this rule before we can just return a copy
-      # of the transitions of this rule. This avoids endless recursions.
-      return rule.transitions.dup unless rule.transitions.empty?
-
-      rule.transitions = []
-      rule.patterns.each do |pat|
-        allTokensOptional = true
-        transitions = { }
-        pat.each do |token|
-          tokenId = token[1..-1]
-          if token[0] == ?!
-            unless @rules.has_key?(tokenId)
-              raise "Fatal Error: Unknown reference to #{tokenId} in pattern " +
-                    "#{pat} + of rule #{rule.name}"
-            end
-            refRule = @rules[tokenId]
-            # If the referenced rule describes optional content, we need to look
-            # at the next token as well.
-            res = getTransitions(@rules[tokenId])
-            allTokensOptional = false unless refRule.optional?(@rules)
-            # Combine the hashes for each pattern into a single hash
-            res.each do |pat_i|
-              pat_i.each { |tok, r| transitions[tok] = r }
-            end
-          elsif '_$.'.include?(token[0])
-            transitions[token] = rule
-            allTokensOptional = false
-          else
-            raise 'Fatal Error: Illegal token type specifier used for token' +
-                  ": #{tokenId}"
-          end
-          break unless allTokensOptional
-        end
-        # Make sure that we only have one possible transition for each
-        # target.
-        transitions.each do |key, value|
-          rule.transitions.each do |trans|
-            if trans.has_key?(key)
-              rule.dump
-              raise "Fatal Error: Rule #{rule.name} has ambiguous " +
-                    "transitions for target #{key}"
-            end
-          end
-        end
-        rule.transitions << transitions
-      end
-      rule.transitions.dup
-    end
-
     def checkRule(rule)
       if rule.patterns.empty?
         raise "Rule #{rule.name} must have at least one pattern"
       end
 
       rule.patterns.each do |pat|
-        pat.each do |tok|
-          type = tok[0]
-          token = tok[1..-1]
-          if type == ?$
-            if @variables.index(token).nil?
+        pat.each do |type, name|
+          if type == :variable
+            if @variables.index(name).nil?
               error('unsupported_token',
-                    "The token #{token} is not supported here.", token[2])
+                    "The token #{name} is not supported here.")
             end
-          elsif type == ?!
-            if @rules[token].nil?
-              raise "Fatal Error: Reference to unknown rule #{token} in " +
+          elsif type == :reference
+            if @rules[name].nil?
+              raise "Fatal Error: Reference to unknown rule #{name} in " +
                     "pattern '#{pat}' of rule #{rule.name}"
             end
           end
@@ -298,235 +262,192 @@ class TaskJuggler
       end
     end
 
-    # This function processes the input starting with the syntax description of
-    # _rule_. It recursively calls this function whenever the syntax description
-    # contains the reference to another rule.
-    # This recursive version has cleaner code and is about 8% faster than
-    # parseRuleNR.
-    def parseRuleR(rule)
-      #Log.enter('parseRuleR', "Parsing with rule #{rule.name}")
-      #puts "Parsing with rule #{rule.name}"
-      result = rule.repeatable ? TextParserResultArray.new : nil
-      # Rules can be marked 'repeatable'. This flag will be set to true after
-      # the first iteration has been completed.
-      repeatMode = false
+    def parseFSM(rule)
+      unless (state = @states[[ rule, nil, 0 ]])
+        error("no_start_state", "No start state for rule #{rule.name} found")
+      end
+      @stack = [ TextParser::StackElement.new(nil, state) ]
+
       loop do
-        # At the beginning of a rule we need a token from the input to determine
-        # which pattern of the rule needs to be processed.
-        token = getNextToken
-
-        return result unless (pattern = findPattern(rule, token, repeatMode))
-        # The @stack will store the resulting value of each element in the
-        # pattern.
-        @stack << TextParser::StackElement.new(pattern.function)
-
-        pattern.each do |element|
-          # Separate the type and token text for pattern element.
-          elType = element[0]
-          elToken = element[1..-1]
-          if elType == ?!
-            # The element is a reference to another rule. Return the token if
-            # we still have one and continue with the referenced rule.
-            unless token.nil?
-              sfi = token[2]
-              returnToken(token)
-              token = nil
+        if state.transitions.empty?
+          # The final states of each pattern have no pre-compiled transitions.
+          # For such a state, we don't need to get a new token.
+          transition = token = nil
+        else
+          transition = state.transition(token = getNextToken)
+        end
+
+        if transition
+          # Shift: This for normal state transitions. This may be from one
+          # token of a pattern to the next token of the same pattern, to the
+          # start of a new pattern or a loop-back to the start of a pattern of
+          # the same rule. The transition tells us what state we have to
+          # process next.
+          state = transition.state
+
+          # If we have looped-back we need to finish the pattern first. Final
+          # tokens of repeatable rules do have transitions!
+          finishPattern(token) if transition.loopBack
+
+          # Transitions that enter rules generate states which we need to
+          # resume at when a rule has been completely processed. We push this
+          # list of states on the @stack.
+          stackElement = @stack.last
+          first = true
+          transition.stateStack.each do |s|
+            if first && s.pattern == stackElement.state.pattern
+              # The first state in the list may just be another state of the
+              # current pattern. In this case, we already have the
+              # StackElement on the @stack. We only need to update the State
+              # for the current StackElement.
+              stackElement.state = s
             else
-              sfi = nil
+              # For other patterns, we just push a new StackElement onto the
+              # @stack.
+              @stack.push(TextParser::StackElement.new(nil, s))
             end
-            @stack.last.store(parseRuleR(@rules[elToken]), sfi)
-            #Log << "Resuming rule #{rule.name}"
-            #puts "Resuming rule #{rule.name}"
-          else
-            # In case the element is a keyword or variable we have to get a new
-            # token if we don't have one anymore.
-            token = getNextToken unless token
-
-            processNormalElements(elType, elToken, token)
+            first = false
+          end
 
-            # The token has been consumed. Reset the variable.
-            token = nil
-            @@expectedTokens = []
+          if state.index == 0
+            # If we have just started with a new pattern (or loop-ed back) we
+            # need to push a new StackEntry onto the @stack. The StackEntry
+            # stores the result of the pattern and keeps the State that we
+            # need to return to in case we jump to other patterns from this
+            # pattern.
+            function = state.index == state.pattern.tokens.length - 1 ?
+                       state.pattern.function : nil
+            @stack.push(TextParser::StackElement.new(state.pattern.function,
+                                                     state))
           end
-        end
 
-        # Once the complete pattern has been processed we call the processing
-        # function for this pattern to operate on the value array. Then pop the
-        # entry for this rule from the stack.
-        @val = @stack.last.val
-        @sourceFileInfo = @stack.last.sourceFileInfo
-        res = nil
-        res = @stack.last.function.call unless @stack.last.function.nil?
-        @stack.pop
-
-        # If the rule is not repeatable we can store the result and break the
-        # outer loop to exit the function.
-        unless rule.repeatable
-          result = res
-          break
+          # Store the token value in the result Array.
+          @stack.last.insert(state.index, token[1], token[2], false)
+        else
+          # Reduce: We've reached the end of a rule. There is no pre-compiled
+          # transition available. The current token, if we have one, is of no
+          # use to us during this state. We just return it to the scanner. The
+          # next state is determined by the first matching state from the
+          # @stack.
+          if state.noReduce
+            # Only states that finish a rule may trigger a reduce operation.
+            # Other states have the noReduce flag set. If a reduce for such a
+            # state is triggered, we found a token that is not supported by
+            # the syntax rules.
+            error("no_reduce",
+                  "Unexpected token '#{token[1]}' found. " +
+                  "Expecting one of " +
+                  "#{@stack.last.state.expectedTokens.join(', ')}")
+          end
+          returnToken(token) if token
+          if finishPattern(token)
+            # Accept: We're done with parsing.
+            break
+          end
+          state = @stack.last.state
         end
-
-        # Otherwise we append the result to the result array and turn repeat
-        # mode on.
-        result << res
-        repeatMode = true
       end
 
-      #Log.exit('parseRuleR', "Finished rule #{rule.name}")
-      #puts "Finished rule #{rule.name}"
-      return result
+      @stack[0].val[0]
     end
 
-    # This function processes the input starting with the syntax description
-    # of _rule_. It's implemented as an unrolled recursion.  It recursively
-    # iterates over the rule tree as controlled by the input file.
-    # This version is not limited by the size of the system stack. So far, I'm
-    # not aware of any project that is too large for the system stack. Since
-    # the recursive version parseRuleR is about 8% faster and has cleaner
-    # code, we use that by default.
-    def parseRuleNR(rule)
-      elementIdx = 0
-      recursionResult = nil
-      # These flags are used to managed the control flow to and from the
-      # recursion point.
-      recur = resume = false
-      # The stack that holds the context for the recursion levels. It's either
-      # just a rule to start a new recursion or an Array of state variables.
-      recursionStack = [ rule ]
-      begin
-        # Pop the top entry from the recursion stack.
-        se = recursionStack.pop
-        if se.is_a?(Array)
-          # We have essentially finished a recursion level and need to get
-          # back to the place where we started the recursion. First, we need
-          # to restore the state again.
-          rule, pattern, elementIdx, result, repeatMode, sfi = se
-          #Log << "Recursion loop started in resume mode for rule #{rule.name}"
-          # Now jump to the recursion point without doing anything else.
-          resume = true
-        else
-          # Start a new recursion level. The rule tells us how to interpret
-          # the input text.
-          rule = se
-          #Log.enter('parseRuleNR', "Parsing with rule #{rule.name}")
-          resume = false
-        end
-
-        unless resume
-          result = rule.repeatable ? TextParserResultArray.new : nil
-          # Rules can be marked 'repeatable'. This flag will be set to true
-          # after the first iteration has been completed.
-          repeatMode = false
+    def finishPattern(token)
+      #dumpStack
+      # To finish a pattern we need to pop the StackElement with the token
+      # values from the stack.
+      stackEntry = @stack.pop
+      if stackEntry.nil? || @stack.empty?
+        # Check if we have reached the bottom of the stack.
+        token = getNextToken unless token
+        if token[0] == :endOfText
+          # If the token is the end of the top-level file, we're done. We push
+          # back the StackEntry since it holds the overall result of the
+          # parsing.
+          @stack.push(stackEntry)
+          return true
         end
+        # If it's not the EOF token, we found a token that violates the syntax
+        # rules.
+        error('unexpctd_token', "Unexpected token '#{token[1]}' found. " +
+              "Expecting one of " +
+              "#{stackEntry.state.expectedTokens.join(', ')}")
+      end
+      # Memorize if the rule for this pattern was repeatable. Then we will
+      # store the result of the pattern in an Array.
+      ruleIsRepeatable = stackEntry.state.rule.repeatable
+
+      state = stackEntry.state
+      result = nil
+      if state.pattern.function
+        # Make the token values and their SourceFileInfo available.
+        @val = stackEntry.val
+        @sourceFileInfo = stackEntry.sourceFileInfo
+        # Now call the pattern action to compute the value of the pattern.
+        result = state.pattern.function.call
+      end
 
-        loop do
-          unless resume
-            # At the beginning of a rule we need a token from the input to
-            # determine which pattern of the rule needs to be processed.
-            token = getNextToken
-
-            break unless (pattern = findPattern(rule, token, repeatMode))
-            # The @stack will store the resulting value of each element in the
-            # pattern.
-            @stack << TextParser::StackElement.new(pattern.function)
-
-            # Once we've found the right pattern, we need to process each
-            # element.
-            elementIdx = 0
-          end
-
-          elementCount = pattern.length
-          while elementIdx < elementCount
-            element = pattern[elementIdx]
-            # Separate the type and token text for pattern element.
-            elType = element[0]
-            elToken = element[1..-1]
-            if elType == ?!
-              unless resume
-                # The element is a reference to another rule. Return the token
-                # if we still have one and continue with the referenced rule.
-                if token
-                  sfi = token[2]
-                  returnToken(token)
-                  token = nil
-                else
-                  sfi = nil
-                end
-                # This is where the recursion would happen. Instead, we push
-                # the state variables and then the next rule onto the
-                # recursion stack.
-                recursionStack.push([ rule, pattern, elementIdx, result,
-                                      repeatMode, sfi ])
-                recursionStack.push(@rules[elToken])
-                # Now terminate all but the outer loops without doing anything
-                # else.
-                recur = true
-                break
-              else
-                # We're back right after where the recursion started. Store
-                # the result and turn resume mode off again.
-                @stack.last.store(recursionResult, sfi)
-                resume = false
-              end
-            else
-              # In case the element is a keyword or variable we have to get a
-              # new token if we don't have one anymore.
-              token = getNextToken unless token
-
-              processNormalElements(elType, elToken, token)
+      # We use the SourceFileInfo of the first token of the pattern to store
+      # it with the result of the pattern.
+      firstSourceFileInfo = stackEntry.firstSourceFileInfo
+      # Store the result at the correct position into the next lower level of
+      # the stack.
+      stackEntry = @stack.last
+      stackEntry.insert(stackEntry.state.index, result,
+                        firstSourceFileInfo, ruleIsRepeatable)
+      false
+    end
 
-              # The token has been consumed. Reset the variable.
-              token = nil
-              @@expectedTokens = []
+    def dumpStack
+      #puts "Stack level #{@stack.length}"
+      @stack.each do |sl|
+        print "#{@stack.index(sl)}: "
+        sl.each do |v|
+          if v.is_a?(Array)
+            begin
+              print "[#{v.join('|')}]|"
+            rescue
+              print "[#{v[0].class}...]|"
+            end
+          else
+            begin
+              print "#{v}|"
+            rescue
+              print v.class
             end
-            elementIdx += 1
-          end # of pattern while loop
-
-          # Skip the rest of the loop in recur mode.
-          break if recur
-
-          elementIdx = 0
-
-          # Once the complete pattern has been processed we call the
-          # processing function for this pattern to operate on the value
-          # array. Then pop the entry for this rule from the stack. The
-          # called function will use @val and @sourceFileInfo to retrieve
-          # data from the parser.
-          @val = @stack.last.val
-          @sourceFileInfo = @stack.last.sourceFileInfo
-          res = @stack.last.function ? @stack.last.function.call : nil
-          @stack.pop
-
-          # If the rule is not repeatable we can store the result and break
-          # the outer loop to exit the function.
-          unless rule.repeatable
-            result = res
-            break
           end
+        end
+        print " -> #{sl.state ? sl.state.to_s(true) : 'nil'} #{sl.function.nil? ? '' : '(Called)'}"
+        puts ""
+      end
+    end
 
-          # Otherwise we append the result to the result array and turn repeat
-          # mode on.
-          result << res
-          # We have completed the first iteration. Set the repeat mode flag to
-          # indicate that further iterations are already re-runs.
-          repeatMode = true
-        end # of rule processing loop
+    # Convert the FSM stack state entries from State objects into [ rule,
+    # pattern, index ] equivalents.
+    def saveFsmStack
+      return unless @stack
 
-        if recur
-          recur = false
-        else
-          #Log.exit('parseRuleNR', "Finished rule #{rule.name}")
-          recursionResult = result
-        end
-      end while !recursionStack.empty?
+      @stack.each do |s|
+        next unless (st = s.state)
+        s.state = [ st.rule, st.pattern, st.index ]
+      end
+    end
+
+    # Convert the FSM stack state entries from [ rule, pattern, index ] into
+    # the respective State objects again.
+    def restoreFsmStack
+      return unless @stack
 
-      return result
+      @stack.each do |s|
+        next unless (state = @states[s.state])
+        raise "Stack restore failed. Cannot find state" unless state
+        s.state = state
+      end
     end
 
     def getNextToken
       token = nextToken
       #Log << "Token: [#{token[0]}][#{token[1]}]"
-      if @badVariables.include?(token[0])
+      if @blockedVariables[token[0]]
         error('unsupported_token',
               "The token #{token[1]} is not supported in this context.",
               token[2])
@@ -534,92 +455,6 @@ class TaskJuggler
       token
     end
 
-    def findPattern(rule, token, repeatMode)
-      # The scanner cannot differentiate between keywords and identifiers.  So
-      # whenever an identifier is returned we have to see if we have a
-      # matching keyword first. If none is found, then look for normal
-      # identifiers.
-      if token[0] == 'ID'
-        if (patIdx = rule.matchingPatternIndex('_' + token[1])).nil?
-          patIdx = rule.matchingPatternIndex("$ID")
-        end
-      elsif token[0] == 'LITERAL'
-        patIdx = rule.matchingPatternIndex('_' + token[1])
-      elsif token[0] == false
-        patIdx = rule.matchingPatternIndex('.')
-      else
-        patIdx = rule.matchingPatternIndex('$' + token[0])
-      end
-
-      # If no matching pattern is found for the token we have to check if the
-      # rule is optional or we are in repeat mode. If this is the case, return
-      # the token back to the scanner. Otherwise, we have found a token we
-      # cannot handle at this point.
-      if patIdx.nil?
-        # Append the list of expected tokens to the @@expectedToken array.
-        # This may be used in a later rule to provide more details when an
-        # error occured.
-        rule.transitions.each do |transition|
-          keys = transition.keys
-          keys.collect! { |key| key[1..-1] }
-          @@expectedTokens += keys
-          @@expectedTokens.sort!
-        end
-
-        unless rule.optional?(@rules) || repeatMode
-          error('unexpctd_token',
-                (token[0] != false ?
-                 "Unexpected token '#{token[1]}' of type " +
-                 "'#{token[0]}'. " :
-                 "Unexpected end of file in #{@scanner.fileName}. ") +
-                (@@expectedTokens.length > 1 ?
-                 "Expecting one of #{@@expectedTokens.join(', ')}" :
-                 "Expecting #{@@expectedTokens[0]}"), token[2])
-        end
-        returnToken(token)
-        return nil
-      end
-
-      rule.pattern(patIdx)
-    end
-
-    # Handle the elements that don't trigger a recursion.
-    def processNormalElements(elType, elToken, token)
-      if elType == ?_
-        # If the element requires a keyword the token must match this
-        # keyword.
-        if elToken != token[1]
-          text = "'#{elToken}' expected but found " +
-                 "'#{token[1]}' (#{token[0]})."
-          unless @@expectedTokens.empty?
-            text = "#{@@expectedTokens.join(', ')} or " + text
-          end
-          error('spec_keywork_expctd', text, token[2])
-        end
-        @stack.last.store(elToken, token[2])
-      elsif elType == ?.
-        if token[0..1] != [ '.', '<END>' ]
-          error('end_expected',
-                "Found garbage at expected end of text: #{token[1]}\n" +
-                "If you see this in the middle of your text, you probably " +
-                "have closed your context too early.", token[2])
-        end
-      else
-        # The token must match the expected variable type.
-        if token[0] != elToken
-          text = "'#{elToken}' expected but found " +
-                 "'#{token[1]}' (#{token[0]})."
-          unless @@expectedTokens.empty?
-            text = "#{@@expectedTokens.join(', ')} or " + text
-          end
-          error('spec_token_expctd', text, token[2])
-        end
-        # If the element is a variable store the value of the token.
-        @stack.last.store(token[1], token[2])
-      end
-    end
-
   end
 
 end
-