taskjuggler 0.0.2

data/lib/TextParser.rb

@@ -0,0 +1,416 @@
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+#
+# = TextParser.rb -- The TaskJuggler III Project Management Software
+#
+# Copyright (c) 2006, 2007, 2008, 2009 by Chris Schlaeger <cs@kde.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+
+require 'TextParser/Pattern'
+require 'TextParser/Rule'
+require 'TextParser/StackElement'
+require 'TjException'
+require 'Log'
+
+class TaskJuggler
+
+  # The TextParser implements a regular LALR parser. But it uses a recursive
+  # rule traversor instead of the more commonly found state machine generated by
+  # yacc-like tools. Since stack depths is not really an issue with a Ruby
+  # implementation this approach has one big advantage. The syntax of the parser
+  # can be modified during parsing. This allows support for languages that can
+  # extend themself. The TaskJuggler syntax is such an beast. Traditional yacc
+  # generated parsers would fail with such a syntax.
+  #
+  # This class is just a base class. A complete parser would derive from this
+  # class and implement the rule set and the functions _nextToken()_ and
+  # _returnToken()_. It also needs to set the array _variables_ to declare all
+  # variables ($SOMENAME) that the scanner may deliver.
+  #
+  # To describe the syntax the functions TextParser#pattern, TextParser#optional
+  # and TextParser#repeatable can be used. When the rule set is changed during
+  # parsing, TextParser#updateParserTables must be called to make the changes
+  # effective. The parser can also document the syntax automatically. To
+  # document a pattern, the functions TextParser#doc, TextParser#descr,
+  # TextParser#also and TextParser#arg can be used.
+  #
+  # To start parsing the input the function TextParser#parse needs to be called
+  # with the name of the start rule.
+  class TextParser
+
+    # Utility class so that we can distinguish Array results from the Array
+    # containing the results of a repeatable rule.
+    class TextParserResultArray < Array
+
+      def initialize
+        super
+      end
+
+      # If there is a repeatable rule that contains another repeatable loop, the
+      # result of the inner rule is an Array that gets put into another Array by
+      # the outer rule. In this case, the inner Array can be merged with the
+      # outer Array.
+      def <<(arg)
+        if arg.is_a?(TextParserResultArray)
+          self.concat(arg)
+        else
+          super
+        end
+      end
+
+    end
+
+    attr_reader :rules
+
+    # Create a new TextParser object.
+    def initialize
+      @rules = { }
+      # Array to hold the token types that the scanner can return.
+      @variables = []
+      # The currently processed rule.
+      @cr = nil
+      # If set to a value larger than 0 debug output will be generated.
+      @@debug = 30
+    end
+
+    # Call all methods that start with 'rule_' to initialize the rules.
+    def initRules
+      methods.each do |m|
+        if m[0, 5] == 'rule_'
+          # Create a new rule with the suffix of the function name as name.
+          newRule(m[5..-1])
+          # Call the function.
+          send(m)
+        end
+      end
+    end
+
+    # Add a new rule to the rule set. _name_ must be a unique identifier. The
+    # function also sets the class variable @cr to the new rule. Subsequent
+    # calls to TextParser#pattern, TextParser#optional or
+    # TextParser#repeatable will then implicitely operate on the most recently
+    # added rule.
+    def newRule(name)
+      raise "Fatal Error: Rule #{name} already exists" if @rules.has_key?(name)
+
+      @rules[name] = @cr = TextParser::Rule.new(name)
+    end
+
+    # Add a new pattern to the most recently added rule. _tokens_ is an array of
+    # strings that specify the syntax elements of the pattern. Each token must
+    # start with an character that identifies the type of the token. The
+    # following types are supported.
+    #
+    # * ! a reference to another rule
+    # * $ a variable token as delivered by the scanner
+    # * _ a literal token.
+    #
+    # _func_ is a Proc object that is called whenever the parser has completed
+    # the processing of this rule.
+    def pattern(tokens, func = nil)
+      @cr.addPattern(TextParser::Pattern.new(tokens, func))
+    end
+
+    # Identify the patterns of the most recently added rule as optional syntax
+    # elements.
+    def optional
+      @cr.setOptional
+    end
+
+    # Identify the patterns of the most recently added rule as repeatable syntax
+    # elements.
+    def repeatable
+      @cr.setRepeatable
+    end
+
+    # This function needs to be called whenever new rules or patterns have been
+    # added and before the next call to TextParser#parse.
+    def updateParserTables
+      @rules.each_value { |rule| rule.transitions = {} }
+      @rules.each_value do |rule|
+        getTransitions(rule)
+        checkRule(rule)
+      end
+    end
+
+    # To parse the input this function needs to be called with the name of the
+    # rule to start with. It returns the result of the processing function of
+    # the top-level parser rule that was specified by _ruleName_.
+    def parse(ruleName, enforceEndOfFile = true)
+      @stack = []
+      @@expectedTokens = []
+      updateParserTables
+      begin
+        result = parseRule(@rules[ruleName])
+      rescue TjException
+        # error('parse_error', $!.message)
+        return nil
+      end
+
+      result
+    end
+
+    # Return the SourceFileInfo of the TextScanner at the beginning of the
+    # currently processed TextParser::Rule. Or return nil if we don't have a
+    # current position.
+    def sourceFileInfo
+      return nil if @stack.empty?
+      @stack.last.sourceFileInfo
+    end
+
+    def matchingRules(keyword)
+      matches = []
+      @rules.each do |name, rule|
+        patIdx = rule.matchingPatternIndex('_' + keyword)
+        matches << [ rule, patIdx ] if patIdx
+      end
+      matches
+    end
+
+    def error(id, text, property = nil)
+      @scanner.error(id, text, property)
+    end
+
+  private
+
+    # getTransitions recursively determines all possible target tokens
+    # that the _rule_ matches. A target token can either be a fixed token
+    # (prefixed with _), a variable token (prefixed with $) or an end token
+    # (just a .). The list of found target tokens is stored in the _transitions_
+    # list of the rule. For each rule pattern we store the transitions for this
+    # pattern in a token -> rule hash.
+    def getTransitions(rule)
+      # If we have processed this rule before we can just return a copy
+      # of the transitions of this rule. This avoids endless recursions.
+      return rule.transitions.clone unless rule.transitions.empty?
+
+      rule.transitions = []
+      rule.patterns.each do |pat|
+        allTokensOptional = true
+        transitions = { }
+        pat.each do |token|
+          tokenId = token[1..-1]
+          if token[0] == ?!
+            unless @rules.has_key?(tokenId)
+              raise "Fatal Error: Unknown reference to #{tokenId} in pattern " +
+                  "#{pat} + of rule #{rule.name}"
+            end
+            refRule = @rules[tokenId]
+            # If the referenced rule describes optional content, we need to look
+            # at the next token as well.
+            res = getTransitions(@rules[tokenId])
+            allTokensOptional = false unless refRule.optional?(@rules)
+            # Combine the hashes for each pattern into a single hash
+            res.each do |pat_i|
+              pat_i.each { |tok, r| transitions[tok] = r }
+            end
+            optional = true if refRule.optional
+          elsif '_$.'.include?(token[0])
+            transitions[token] = rule
+            allTokensOptional = false
+          else
+            raise 'Fatal Error: Illegal token type specifier used for token' +
+                  ": #{tokenId}"
+          end
+          break unless allTokensOptional
+        end
+        # Make sure that we only have one possible transition for each
+        # target.
+        transitions.each do |key, value|
+          rule.transitions.each do |trans|
+            if trans.has_key?(key)
+              raise "Fatal Error: Rule #{rule.name} has ambigeous " +
+                    "transitions for target #{key}"
+            end
+          end
+        end
+        rule.transitions << transitions
+      end
+      rule.transitions.clone
+    end
+
+    def checkRule(rule)
+      if rule.patterns.empty?
+        raise "Rule #{rule.name} must have at least one pattern"
+      end
+
+      rule.patterns.each do |pat|
+        pat.each do |tok|
+          type = tok[0]
+          token = tok[1..-1]
+          if type == ?$
+            if @variables.index(token).nil?
+              raise "Fatal Error: Illegal variable type #{token} used for " +
+                    "rule #{rule.name} in pattern '#{pat}'"
+            end
+          elsif type == ?!
+            if @rules[token].nil?
+              raise "Fatal Error: Reference to unknown rule #{token} in " +
+                    "pattern '#{pat}' of rule #{rule.name}"
+            end
+          end
+        end
+      end
+    end
+
+    # This function processes the input starting with the syntax description of
+    # _rule_. It recursively calls this function whenever the syntax description
+    # contains the reference to another rule.
+    def parseRule(rule)
+      Log.enter('parseRule', "Parsing with rule #{rule.name}")
+      result = rule.repeatable ? TextParserResultArray.new : nil
+      # Rules can be marked 'repeatable'. This flag will be set to true after
+      # the first iteration has been completed.
+      repeatMode = false
+      loop do
+        # At the beginning of a rule we need a token from the input to determine
+        # which pattern of the rule needs to be processed.
+        begin
+          token = nextToken
+          Log << "Token: [#{token[0]}][#{token[1]}]"
+        rescue TjException
+          error('parse_rule1', $!.message)
+        end
+
+        # The scanner cannot differentiate between keywords and identifiers. So
+        # whenever an identifier is returned we have to see if we have a
+        # matching keyword first. If none is found, then look for normal
+        # identifiers.
+        if token[0] == 'ID'
+          if (patIdx = rule.matchingPatternIndex('_' + token[1])).nil?
+            patIdx = rule.matchingPatternIndex("$ID")
+          end
+        elsif token[0] == 'LITERAL'
+          patIdx = rule.matchingPatternIndex('_' + token[1])
+        elsif token[0] == false
+          patIdx = rule.matchingPatternIndex('.')
+        else
+          patIdx = rule.matchingPatternIndex('$' + token[0])
+        end
+
+        # If no matching pattern is found for the token we have to check if the
+        # rule is optional or we are in repeat mode. If this is the case, return
+        # the token back to the scanner. Otherwise we have found a token we
+        # cannot handle at this point.
+        if patIdx.nil?
+          # Append the list of expected tokens to the @@expectedToken array.
+          # This may be used in a later rule to provide more details when an
+          # error occured.
+          rule.transitions.each do |transition|
+            keys = transition.keys
+            keys.collect! { |key| key[1..-1] }
+            @@expectedTokens += keys
+            @@expectedTokens.sort!
+          end
+
+          unless rule.optional?(@rules) || repeatMode
+            error('unexpctd_token',
+                  (token[0] != false ?
+                   "Unexpected token '#{token[1]}' of type '#{token[0]}'. " :
+                   "Unexpected end of file in #{@scanner.fileName}. ") +
+                  (@@expectedTokens.length > 1 ?
+                   "Expecting one of #{@@expectedTokens.join(', ')}" :
+                   "Expecting #{@@expectedTokens[0]}"))
+          end
+          returnToken(token)
+          Log.exit('parseRule', "Finished rule #{rule.name}")
+          return result
+        end
+
+        pattern = rule.pattern(patIdx)
+        @stack << TextParser::StackElement.new(rule, pattern.function,
+                                               @scanner.sourceFileInfo)
+
+        pattern.each do |element|
+          # Separate the type and token text for pattern element.
+          elType = element[0]
+          elToken = element[1..-1]
+          if elType == ?!
+            # The element is a reference to another rule. Return the token if we
+            # still have one and continue with the referenced rule.
+            unless token.nil?
+              returnToken(token)
+              token = nil
+            end
+            @stack.last.store(parseRule(@rules[elToken]))
+          else
+            # In case the element is a keyword or variable we have to get a new
+            # token if we don't have one anymore.
+            if token.nil?
+              begin
+                token = nextToken
+                Log << "Token: [#{token[0]}][#{token[1]}]"
+              rescue TjException
+                error('parse_rule2', $!.message)
+              end
+            end
+
+            if elType == ?_
+              # If the element requires a keyword the token must match this
+              # keyword.
+              if elToken != token[1]
+                text = "'#{elToken}' expected but found " +
+                       "'#{token[1]}' (#{token[0]})."
+                unless @@expectedTokens.empty?
+                  text = "#{@@expectedTokens.join(', ')} or " + text
+                end
+                error('spec_keywork_expctd', text)
+              end
+              @stack.last.store(elToken)
+            elsif elType == ?.
+              if token != [ '.', '<END>' ]
+                error('end_expected', 'End expected but found ' +
+                      "'#{token[1]}' (#{token[0]}).")
+              end
+            else
+              # The token must match the expected variable type.
+              if token[0] != elToken
+                text = "'#{elToken}' expected but found " +
+                       "'#{token[1]}' (#{token[0]})."
+                unless @@expectedTokens.empty?
+                  text = "#{@@expectedTokens.join(', ')} or " + text
+                end
+                error('spec_token_expctd', text)
+              end
+              # If the element is a variable store the value of the token.
+              @stack.last.store(token[1])
+            end
+            # The token has been consumed. Reset the variable.
+            token = nil
+            @@expectedTokens = []
+          end
+        end
+
+        # Once the complete pattern has been processed we call the processing
+        # function for this pattern to operate on the value array. Then pop the
+        # entry for this rule from the stack.
+        @val = @stack.last.val
+        res = nil
+        res = @stack.last.function.call unless @stack.last.function.nil?
+        @stack.pop
+
+        # If the rule is not repeatable we can store the result and break the
+        # outer loop to exit the function.
+        unless rule.repeatable
+          result = res
+          break
+        end
+
+        # Otherwise we append the result to the result array and turn repeat
+        # mode on.
+        result << res
+        repeatMode = true
+      end
+
+      Log.exit('parseRule', "Finished rule #{rule.name}")
+      return result
+    end
+
+  end
+
+end
+

data/lib/TextParser/Pattern.rb

@@ -0,0 +1,263 @@
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+#
+# = TextParser/Pattern.rb -- The TaskJuggler III Project Management Software
+#
+# Copyright (c) 2006, 2007, 2008, 2009 by Chris Schlaeger <cs@kde.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+
+require 'TextParser/TokenDoc'
+
+class TaskJuggler::TextParser
+
+  # This class models the most crutial elements of a syntax description - the
+  # pattern. A TextParserPattern primarily consists of a set of tokens. Tokens
+  # are Strings where the first character determines the type of the token.
+  # There are 4 known types.
+  #
+  # Terminal token: The terminal token is prefixed by an underscore. Terminal
+  # tokens are terminal symbols of the syntax tree. They just represent
+  # themselves.
+  #
+  # Variable token: The variable token describes values of a certain class such
+  # as strings or numbers. The token is prefixed by a dollar sign and the text
+  # of the token specifies the variable type. See ProjectFileParser for a
+  # complete list of variable types.
+  #
+  # Reference token: The reference token specifies a reference to another parser
+  # rule. The token is prefixed by a bang and the text matches the name of the
+  # rule. See TextParserRule for details.
+  #
+  # End token: The . token marks the expected end of the input stream.
+  #
+  # In addition to the pure syntax tree information the pattern also holds
+  # documentary information about the pattern.
+  class Pattern
+
+    attr_reader :keyword, :doc, :seeAlso, :exampleFile, :exampleTag,
+                :tokens, :function
+
+    def initialize(tokens, function = nil)
+      # A unique name for the pattern that is used in the documentation.
+      @keyword = nil
+      # Initialize pattern doc as empty.
+      @doc = nil
+      # A list of TokenDoc elements that describe the meaning of variable
+      # tokens. The order of the tokens and entries in the Array must correlate.
+      @args = []
+      # A list of references to other patterns that are related to this pattern.
+      @seeAlso = []
+      # A reference to a file under test/TestSuite/Syntax/Correct and a tag
+      # within that file. This identifies example TJP code to be included with
+      # the reference manual.
+      @exampleFile = nil
+      @exampleTag = nil
+
+      tokens.each do |token|
+        unless '!$_.'.include?(token[0])
+          raise "Fatal Error: All pattern tokens must start with a type " +
+                "identifier [!$_.]: #{tokens.join(', ')}"
+        end
+        # Initialize pattern argument descriptions as empty.
+        @args << nil
+      end
+      @tokens = tokens
+      @function = function
+      # In some cases we don't want to show all tokens in the syntax
+      # documentation. This value specifies the index of the last shown token.
+      @lastSyntaxToken = @tokens.length - 1
+    end
+
+    # Set the keyword and documentation text for the pattern.
+    def setDoc(keyword, doc)
+      @keyword = keyword
+      @doc = doc
+    end
+
+    # Set the documentation text and for the idx-th variable.
+    def setArg(idx, doc)
+      @args[idx] = doc
+    end
+
+    # Restrict the syntax documentation to the first +idx+ tokens.
+    def setLastSyntaxToken(idx)
+      @lastSyntaxToken = idx
+    end
+
+    # Set the references to related patterns.
+    def setSeeAlso(also)
+      @seeAlso = also
+    end
+
+    # Set the file and tag for the TJP code example.
+    def setExample(file, tag)
+      @exampleFile = file
+      @exampleTag = tag
+    end
+
+    # Conveniance function to access individual tokens by index.
+    def [](i)
+      @tokens[i]
+    end
+
+    # Iterator for tokens.
+    def each
+      @tokens.each { |tok| yield tok }
+    end
+
+    # Returns true of the pattern is empty.
+    def empty?
+      @tokens.empty?
+    end
+
+    # Returns the number of tokens in the pattern.
+    def length
+      @tokens.length
+    end
+
+    # Return true if all tokens of the pattern are optional. If a token
+    # references a rule, this rule is followed for the check.
+    def optional?(rules)
+      @tokens.each do |token|
+        if token[0] == ?_ || token[0] == ?$
+          return false
+        elsif token[0] == ?!
+          token = token[1..-1]
+          if !rules[token].optional?(rules)
+            return false
+          end
+        end
+      end
+      true
+    end
+
+    # Returns true if the i-th token is a terminal symbol.
+    def terminalSymbol?(i)
+      @tokens[i][0] == ?$ || @tokens[i][0] == ?_
+    end
+
+    # Find recursively the first terminal token of this pattern. If an index is
+    # specified start the search at this n-th pattern token instead of the
+    # first. The return value is either nil or a [ token, pattern ] tuple.
+    def terminalToken(rules, index = 0)
+      # Terminal token start with an underscore or dollar character.
+      if @tokens[index][0] == ?_ || @tokens[index][0] == ?$
+        return [ @tokens[index].slice(1, @tokens[index].length - 1), self ]
+      elsif @tokens[index][0] == ?!
+        # Token starting with a bang reference another rule. We have to continue
+        # the search at this rule. First, we get rid of the bang to get the rule
+        # name.
+        token = @tokens[index].slice(1, @tokens[index].length - 1)
+        # Then find the rule
+        rule = rules[token]
+        # The rule may only have a single pattern. If not, then this pattern has
+        # no terminal token.
+        return nil if rule.patterns.length != 1
+        return rule.patterns[0].terminalToken(rules)
+      end
+      nil
+    end
+
+    # Returns a string that expresses the elements of the pattern in an EBNF
+    # like fashion. The resolution of the pattern is done recursively. This is
+    # just the wrapper function that sets up the stack.
+    def to_syntax(argDocs, rules, skip = 0)
+      to_syntax_r({}, argDocs, rules, skip)
+    end
+
+    def to_syntax_r(stack, argDocs, rules, skip)
+      # If we find ourself on the stack we hit a recursive pattern. This is used
+      # in repetitions.
+      if stack[self]
+        return '[, ... ]'
+      end
+
+      # "Push" us on the stack.
+      stack[self] = true
+
+      str = ''
+      first = true
+      # Analyze the tokens of the pattern skipping the first 'skip' tokens.
+      skip.upto(@lastSyntaxToken) do |i|
+        token = @tokens[i]
+        # If the first token is a _{ the pattern describes optional attributes.
+        # They are represented by a standard idiom.
+        if first
+          first = false
+          return '{ <attributes> }' if token == '_{'
+        else
+          # Separate the syntax elemens by a whitespace.
+          str << ' '
+        end
+
+        typeId = token[0]
+        token = token.slice(1, token.length - 1)
+
+        if @args[i]
+          # The argument is documented in the syntax definition. We copy the
+          # entry as we need to modify it.
+          argDoc = @args[i].clone
+
+          # A documented argument without a name is a terminal token. We use the
+          # terminal symbol as name.
+          if @args[i].name.nil?
+            str << "#{token}"
+            argDoc.name = token
+          else
+            str << "<#{@args[i].name}>"
+          end
+          addArgDoc(argDocs, argDoc)
+
+          # Documented arguments don't have the type set yet. Use the token
+          # value for that.
+          if typeId == ?$
+            argDoc.typeSpec = "<#{token}>"
+          end
+        else
+          # Undocumented tokens are recursively expanded.
+          case typeId
+          when ?_
+            # Literals are shown as such.
+            str << token
+          when ?$
+            # Variables are enclosed by angle brackets.
+            str << '<' + token + '>'
+          when ?!
+            if rules[token].patterns.length == 1 &&
+               !rules[token].patterns[0].doc.nil?
+              addArgDoc(argDocs, TokenDoc.new(rules[token].patterns[0].keyword,
+                                              rules[token].patterns[0]))
+              str << '<' + rules[token].patterns[0].keyword + '>'
+            else
+              # References are followed recursively.
+              str << rules[token].to_syntax(stack, argDocs, rules, 0)
+            end
+          end
+        end
+      end
+      # Remove us from the "stack" again.
+      stack.delete(self)
+      str
+    end
+
+    def to_s
+      @tokens.join(' ')
+    end
+
+  private
+
+    def addArgDoc(argDocs, argDoc)
+      raise 'Error' if argDoc.name.nil?
+      argDocs.each do |ad|
+        return if ad.name == argDoc.name
+      end
+      argDocs << argDoc
+    end
+
+  end
+
+end