taskjuggler 0.0.7 → 0.0.8

data/lib/TextParser/Pattern.rb

@@ -11,6 +11,7 @@
 #
 
 require 'TextParser/TokenDoc'
+require 'TextParser/State'
 
 class TaskJuggler::TextParser
 
@@ -19,18 +20,18 @@ class TaskJuggler::TextParser
   # are Strings where the first character determines the type of the token.
   # There are 4 known types.
   #
-  # Terminal token: The terminal token is prefixed by an underscore. Terminal
-  # tokens are terminal symbols of the syntax tree. They just represent
-  # themselves.
+  # Terminal token: In the syntax declaration the terminal token is prefixed
+  # by an underscore. Terminal tokens are terminal symbols of the syntax tree.
+  # They just represent themselves.
   #
   # Variable token: The variable token describes values of a certain class such
-  # as strings or numbers. The token is prefixed by a dollar sign and the text
-  # of the token specifies the variable type. See ProjectFileParser for a
-  # complete list of variable types.
+  # as strings or numbers. In the syntax declaration the token is prefixed by
+  # a dollar sign and the text of the token specifies the variable type. See
+  # ProjectFileParser for a complete list of variable types.
   #
   # Reference token: The reference token specifies a reference to another parser
-  # rule. The token is prefixed by a bang and the text matches the name of the
-  # rule. See TextParserRule for details.
+  # rule. In the syntax declaration the token is prefixed by a bang and the
+  # text matches the name of the rule. See TextParserRule for details.
   #
   # End token: The . token marks the expected end of the input stream.
   #
@@ -57,19 +58,125 @@ class TaskJuggler::TextParser
       @exampleFile = nil
       @exampleTag = nil
 
+      @tokens = []
       tokens.each do |token|
         unless '!$_.'.include?(token[0])
           raise "Fatal Error: All pattern tokens must start with a type " +
                 "identifier [!$_.]: #{tokens.join(', ')}"
         end
+        # For the syntax specification using a prefix character is more
+        # convenient. But for further processing, we need to split the string
+        # into two symbols. The prefix determines the token type, the rest is
+        # the token name. There are 4 types of tokens:
+        # :reference : a reference to another rule
+        # :variable : a terminal symbol
+        # :literal : a user defined string
+        # :eof : marks the end of an input stream
+        type = [ :reference, :variable, :literal, :eof ]['!$_.'.index(token[0])]
+        # For literals we use a String to store the token content. For others,
+        # a symbol is better suited.
+        name = type == :literal ?
+               token[1..-1] : (type == :eof ? '<END>' : token[1..-1].intern)
+        # We favor an Array to store the 2 elements over a Hash for
+        # performance reasons.
+        @tokens << [ type, name ]
         # Initialize pattern argument descriptions as empty.
         @args << nil
       end
-      @tokens = tokens
       @function = function
       # In some cases we don't want to show all tokens in the syntax
       # documentation. This value specifies the index of the last shown token.
       @lastSyntaxToken = @tokens.length - 1
+
+      @transitions = []
+    end
+
+    # Generate the state machine states for the pattern. _rule_ is the Rule
+    # that the pattern belongs to. A list of generated State objects will be
+    # returned.
+    def generateStates(rule)
+      states = []
+      @tokens.length.times { |i| states << State.new(rule, self, i) }
+      states
+    end
+
+    # Add the transitions to the State objects of this pattern. _states_ is a
+    # Hash with all State objects. _rules_ is a Hash with the Rule objects of
+    # the syntax. _stateStack_ is an Array of State objects that have been
+    # traversed before reaching this pattern. _sourceState_ is the State that
+    # the transition originates from. _destRule_, this pattern and _destIndex_
+    # describe the State the transition is leading to. _loopBack_ is boolean
+    # flag, set to true when the transition describes a loop back to the start
+    # of the Rule.
+    def addTransitionsToState(states, rules, stateStack, sourceState,
+                              destRule, destIndex, loopBack)
+      # If we hit a token in the pattern that is optional, we need to consider
+      # the next token of the pattern as well.
+      loop do
+        if destIndex >= @tokens.length
+          # Have we reached the end of the pattern? Such state always trigger
+          # a reduce operation.
+          sourceState.noReduce = false
+          if sourceState.rule == destRule
+            if destRule.repeatable
+              # The transition leads us back to the start of the Rule. This
+              # will generate transitions to the first token of all patterns
+              # of this Rule.
+              destRule.addTransitionsToState(states, rules, [], sourceState,
+                                             true)
+            end
+          end
+          # We've reached the end of the pattern. No more transitions to
+          # consider.
+          return
+        end
+
+        # The token descriptor tells us where the transition(s) need to go to.
+        tokenType, tokenName = token = @tokens[destIndex]
+
+        case tokenType
+        when :reference
+          # The descriptor references another rule.
+          unless (refRule = rules[tokenName])
+            raise "Unknown rule #{tokenName} referenced in rule #{refRule.name}"
+          end
+          # If we reference another rule from a pattern, we need to come back
+          # to the pattern once we are done with the referenced rule. To be
+          # able to come back, we collect a list of all the States that we
+          # have passed during a reference resolution. This list forms a stack
+          # that is popped during recude operations of the parser FSM.
+          skippedState = states[[ destRule, self, destIndex ]]
+          # Rules may reference themselves directly or indirectly. To avoid
+          # endless recursions of this algorithm, we stop once we have
+          # detected a recursion. We have already all necessary transitions
+          # collected. The recursion will be unrolled in the parser FSM.
+          unless stateStack.include?(skippedState)
+            # Push the skipped state on the stateStack before recursing.
+            stateStack.push(skippedState)
+            refRule.addTransitionsToState(states, rules, stateStack,
+                                          sourceState, loopBack)
+            # Once we're done, remove the State from the stateStack again.
+            stateStack.pop
+          end
+
+          # If the referenced rule is not optional, we have no further
+          # transitions for this pattern at this destIndex.
+          break unless refRule.optional?(rules)
+        else
+          unless (destState = states[[ destRule, self, destIndex ]])
+            raise "Destination state not found"
+          end
+          # We've found a transition to a terminal token. Add the transition
+          # to the source State.
+          sourceState.addTransition(@tokens[destIndex], destState, stateStack,
+                                    loopBack)
+          # Fixed tokens are never optional. There are no more transitions for
+          # this pattern at this index.
+          break
+        end
+
+        destIndex += 1
+      end
     end
 
     # Set the keyword and documentation text for the pattern.
@@ -106,7 +213,7 @@ class TaskJuggler::TextParser
 
     # Iterator for tokens.
     def each
-      @tokens.each { |tok| yield tok }
+      @tokens.each { |type, name| yield(type, name) }
     end
 
     # Returns true of the pattern is empty.
@@ -122,12 +229,11 @@ class TaskJuggler::TextParser
     # Return true if all tokens of the pattern are optional. If a token
     # references a rule, this rule is followed for the check.
     def optional?(rules)
-      @tokens.each do |token|
-        if token[0] == ?_ || token[0] == ?$
+      @tokens.each do |type, name|
+        if type == :literal || type == :variable
           return false
-        elsif token[0] == ?!
-          token = token[1..-1]
-          if !rules[token].optional?(rules)
+        elsif type == :reference
+          if !rules[name].optional?(rules)
             return false
           end
         end
@@ -137,32 +243,29 @@ class TaskJuggler::TextParser
 
     # Returns true if the i-th token is a terminal symbol.
     def terminalSymbol?(i)
-      @tokens[i][0] == ?$ || @tokens[i][0] == ?_
+      @tokens[i][0] == :variable || @tokens[i][0] == :literal
     end
 
     # Find recursively the first terminal token of this pattern. If an index is
     # specified start the search at this n-th pattern token instead of the
     # first. The return value is an Array of [ token, pattern ] tuple.
     def terminalTokens(rules, index = 0)
+      type, name = @tokens[index]
       # Terminal token start with an underscore or dollar character.
-      if @tokens[index][0] == ?_
-        return [ [ @tokens[index][1..-1], self ] ]
-      elsif @tokens[index][0] == ?$
+      if type == :literal
+        return [ [ name, self ] ]
+      elsif type == :variable
         return []
-      elsif @tokens[index][0] == ?!
-        # Token starting with a bang reference another rule. We have to
-        # continue the search at this rule. First, we get rid of the bang to
-        # get the rule name.
-        token = @tokens[index][1..-1]
-        # Then find the rule
-        rule = rules[token]
+      elsif type == :reference
+        # We have to continue the search at this rule.
+        rule = rules[name]
         # The rule may only have a single pattern. If not, then this pattern
         # has no terminal token.
         tts = []
         rule.patterns.each { |p| tts += p.terminalTokens(rules, 0) }
         return tts
       else
-        raise "Unexpected token #{tokens[index]}"
+        raise "Unexpected token #{type} #{name}"
       end
     end
 
@@ -173,6 +276,7 @@ class TaskJuggler::TextParser
       to_syntax_r({}, argDocs, rules, skip)
     end
 
+    # Generate a syntax description for this pattern.
     def to_syntax_r(stack, argDocs, rules, skip)
       # If we find ourself on the stack we hit a recursive pattern. This is used
       # in repetitions.
@@ -187,20 +291,17 @@ class TaskJuggler::TextParser
       first = true
       # Analyze the tokens of the pattern skipping the first 'skip' tokens.
       skip.upto(@lastSyntaxToken) do |i|
-        token = @tokens[i]
+        type, name = @tokens[i]
         # If the first token is a _{ the pattern describes optional attributes.
         # They are represented by a standard idiom.
         if first
           first = false
-          return '{ <attributes> }' if token == '_{'
+          return '{ <attributes> }' if name == '{'
         else
           # Separate the syntax elemens by a whitespace.
           str << ' '
         end
 
-        typeId = token[0]
-        token = token.slice(1, token.length - 1)
-
         if @args[i]
           # The argument is documented in the syntax definition. We copy the
           # entry as we need to modify it.
@@ -209,8 +310,8 @@ class TaskJuggler::TextParser
           # A documented argument without a name is a terminal token. We use the
           # terminal symbol as name.
           if @args[i].name.nil?
-            str << "#{token}"
-            argDoc.name = token
+            str << "#{name}"
+            argDoc.name = name
           else
             str << "<#{@args[i].name}>"
           end
@@ -218,27 +319,27 @@ class TaskJuggler::TextParser
 
           # Documented arguments don't have the type set yet. Use the token
           # value for that.
-          if typeId == ?$
-            argDoc.typeSpec = "<#{token}>"
+          if type == :variable
+            argDoc.typeSpec = "<#{name}>"
           end
         else
           # Undocumented tokens are recursively expanded.
-          case typeId
-          when ?_
+          case type
+          when :literal
             # Literals are shown as such.
-            str << token
-          when ?$
+            str << name.to_s
+          when :variable
             # Variables are enclosed by angle brackets.
-            str << '<' + token + '>'
-          when ?!
-            if rules[token].patterns.length == 1 &&
-               !rules[token].patterns[0].doc.nil?
-              addArgDoc(argDocs, TokenDoc.new(rules[token].patterns[0].keyword,
-                                              rules[token].patterns[0]))
-              str << '<' + rules[token].patterns[0].keyword + '>'
+            str << "<#{name}>"
+          when :reference
+            if rules[name].patterns.length == 1 &&
+               !rules[name].patterns[0].doc.nil?
+              addArgDoc(argDocs, TokenDoc.new(rules[name].patterns[0].keyword,
+                                              rules[name].patterns[0]))
+              str << '<' + rules[name].patterns[0].keyword + '>'
             else
               # References are followed recursively.
-              str << rules[token].to_syntax(stack, argDocs, rules, 0)
+              str << rules[name].to_syntax(stack, argDocs, rules, 0)
             end
           end
         end
@@ -248,8 +349,26 @@ class TaskJuggler::TextParser
       str
     end
 
+    # Generate a text form of the pattern. This is similar to the syntax in
+    # the original syntax description.
     def to_s
-      @tokens.join(' ')
+      str = ""
+      @tokens.each do |type, name|
+        case type
+        when :reference
+          str += "!#{name} "
+        when :variable
+          str += "$#{name } "
+        when :literal
+          str += "#{name} "
+        when :eof
+          str += ". "
+        else
+          raise "Unknown type #{type}"
+        end
+      end
+
+      str
     end
 
   private

data/lib/TextParser/Rule.rb

@@ -10,6 +10,8 @@
 # published by the Free Software Foundation.
 #
 
+require 'TextParser/State'
+
 class TaskJuggler::TextParser
 
   # The TextParserRule holds the basic elment of the syntax description. Each
@@ -25,7 +27,6 @@ class TaskJuggler::TextParser
   class Rule
 
     attr_reader :name, :patterns, :optional, :repeatable, :keyword, :doc
-    attr_accessor :transitions
 
     # Create a new syntax rule called +name+.
     def initialize(name)
@@ -33,11 +34,16 @@ class TaskJuggler::TextParser
       @patterns = []
       @repeatable = false
       @optional = false
-      @transitions = []
-      # In case a rule is optional or any of the patterns is fully optional,
-      # this variable is set to true.
-      @transitiveOptional = nil
       @keyword = nil
+
+      flushCache
+    end
+
+    def flushCache
+      # A rule is considered to describe optional tokens in case the @optional
+      # flag is set or all of the patterns reference optional rules again.
+      # This variable caches the transitively determined optional value.
+      @transitiveOptional = nil
     end
 
     # Add a new +pattern+ to the Rule. It should be of type
@@ -71,6 +77,28 @@ class TaskJuggler::TextParser
       end
     end
 
+    def generateStates
+      # First, add an entry State for this rule. Entry states are never
+      # reached by normal state transitions. They are only used as (re-)start
+      # states.
+      states = [ State.new(self) ]
+
+      @patterns.each do |pattern|
+        states += pattern.generateStates(self)
+      end
+      states
+    end
+
+    # Return a Hash of all state transitions caused by the 1st token of each
+    # pattern of this rule.
+    def addTransitionsToState(states, rules, stateStack, sourceState,
+                              loopBack)
+      @patterns.each do |pattern|
+        pattern.addTransitionsToState(states, rules, stateStack.dup, sourceState,
+                                      self, 0, loopBack)
+      end
+    end
+
     # Mark the syntax element described by this Rule as a repeatable element
     # that can occur once or more times in sequence.
     def setRepeatable
@@ -117,18 +145,6 @@ class TaskJuggler::TextParser
       @patterns[idx]
     end
 
-    # Return the pattern of this rule that matches the given +token+. If no
-    # pattern matches, return nil.
-    def matchingPatternIndex(token)
-      i = 0
-      @transitions.each do |t|
-        return i if t.has_key?(token)
-        i += 1
-      end
-
-      nil
-    end
-
     def to_syntax(stack, docs, rules, skip)
       str = ''
       str << '[' if @optional || @repeatable

data/lib/TextParser/StackElement.rb

@@ -10,6 +10,8 @@
 # published by the Free Software Foundation.
 #
 
+require 'SourceFileInfo'
+
 class TaskJuggler::TextParser
 
   # This class models the elements of the stack that the TextParser uses to keep
@@ -18,22 +20,52 @@ class TaskJuggler::TextParser
   # also store the function that must be called to store the collected values.
   class StackElement
 
-    attr_reader :val, :rule, :function, :sourceFileInfo
+    attr_reader :val, :function, :sourceFileInfo, :firstSourceFileInfo
+    attr_accessor :state
 
     # Create a new stack element. _rule_ is the TextParserRule that triggered
     # the creation of this element. _function_ is the function that will be
     # called at the end to store the collected data. _sourceFileInfo_ is a
     # SourceFileInfo reference that describes the TextScanner position when the
     # rule was entered.
-    def initialize(function)
+    def initialize(function, state = nil)
       # This Array stores the collected values.
       @val = []
       # Array to store the source file references for the collected values.
       @sourceFileInfo = []
+      # A shortcut to the first non-nil sourceFileInfo.
+      @firstSourceFileInfo = nil
       # Counter used for StackElement::store()
       @position = 0
       # The method that will process the collected values.
       @function = function
+      @state = state
+    end
+
+    # Insert the value _val_ at the position _index_. It also stores the
+    # _sourceFileInfo_ for this element. In case _multiValue_ is true, the
+    # old value is not overwritten, but values are stored in an
+    # TextParserResultArray object.
+    def insert(index, val, sourceFileInfo, multiValue)
+      if multiValue
+        if @val[index]
+          # We already have a value for this token position.
+          unless @val[index].is_a?(TextParserResultArray)
+            # This should never happen.
+            raise "#{@val[index].class} must be an Array"
+          end
+        else
+          @val[index] = TextParserResultArray.new
+        end
+        # Just append the value and apply the special Array merging.
+        @val[index] << val
+      else
+        @val[index] = val
+      end
+      @sourceFileInfo[index] = sourceFileInfo
+      # Store the first SFI for faster access.
+      @firstSourceFileInfo = sourceFileInfo unless @firstSourceFileInfo
+      val
     end
 
     # Store a collected value and move the position to the next pattern.
@@ -43,6 +75,14 @@ class TaskJuggler::TextParser
       @position += 1
     end
 
+    def each
+      @val.each { |x| yield x }
+    end
+
+    def length
+      @val.length
+    end
+
   end
 
 end