taskjuggler 0.0.5 → 0.0.6

data/lib/TextParser/Rule.rb

@@ -120,8 +120,10 @@ class TaskJuggler::TextParser
     # Return the pattern of this rule that matches the given +token+. If no
     # pattern matches, return nil.
     def matchingPatternIndex(token)
-      @transitions.length.times do |i|
-        return i if @transitions[i].has_key?(token)
+      i = 0
+      @transitions.each do |t|
+        return i if t.has_key?(token)
+        i += 1
       end
 
       nil

data/lib/TextParser/StackElement.rb

@@ -25,20 +25,22 @@ class TaskJuggler::TextParser
     # called at the end to store the collected data. _sourceFileInfo_ is a
     # SourceFileInfo reference that describes the TextScanner position when the
     # rule was entered.
-    def initialize(rule, function, sourceFileInfo)
+    def initialize(function)
       # This Array stores the collected values.
       @val = []
+      # Array to store the source file references for the collected values.
+      @sourceFileInfo = []
+      # Counter used for StackElement::store()
       @position = 0
-      @rule = rule
+      # The method that will process the collected values.
       @function = function
-      @sourceFileInfo = sourceFileInfo
     end
 
     # Store a collected value and move the position to the next pattern.
     def store(val, sourceFileInfo = nil)
       @val[@position] = val
+      @sourceFileInfo[@position] = sourceFileInfo
       @position += 1
-      @sourceFileInfo = sourceFileInfo if sourceFileInfo
     end
 
   end

data/lib/TextScanner.rb

@@ -10,6 +10,9 @@
 # published by the Free Software Foundation.
 #
 
+require 'stringio'
+require 'strscan'
+
 require 'UTF8String'
 require 'TjTime'
 require 'TjException'
@@ -20,10 +23,27 @@ require 'Log'
 
 class TaskJuggler
 
-  # The TextScanner class can scan text files and chop then into tokens to be
-  # used by a parser. Files can be nested. A file can include an other file.
+  # The TextScanner class is an abstract text scanner with support for nested
+  # include files and text macros. The tokenizer will operate on rules that
+  # must be provided by a derived class. The scanner is modal. Each mode
+  # operates only with the subset of token patterns that are assigned to the
+  # current mode. The current line is tracked accurately and can be used for
+  # error reporting. The scanner can operate on Strings or Files.
   class TextScanner
 
+    class MacroStackEntry
+
+      attr_reader :macro, :args, :text, :endPos
+
+      def initialize(macro, args, text, endPos)
+        @macro = macro
+        @args = args
+        @text = text
+        @endPos = endPos
+      end
+
+    end
+
     # This class is used to handle the low-level input operations. It knows
     # whether it deals with a text buffer or a file and abstracts this to the
     # TextScanner. For each nested file the scanner puts an StreamHandle on the
@@ -32,21 +52,102 @@ class TaskJuggler
     # completely processed.
     class StreamHandle
 
-      attr_accessor :lineNo, :columnNo, :line, :charBuffer
-      attr_reader :fileName
+      attr_reader :fileName, :macroStack
 
       def initialize
-        @lineNo = 1
-        @columnNo = 1
-        @line = ""
-        @charBuffer = []
         @fileName = nil
+        @stream = nil
+        @line = nil
+        @pos = 0
+        @endPos = 1
+        @scanner = nil
+        @wrapped = false
+        @macroStack = []
+        @nextMacroEnd = nil
+      end
+
+      def close
+        @stream = nil
+      end
+
+      def injectMacro(macro, args, text)
+        @nextMacroEnd = @pos + text.length
+        @line = @line[0, @pos] + text + @line[@pos..-1]
+        @scanner = StringScanner.new(@line)
+        @scanner.pos = @pos
+
+        # Simple detection for recursive macro calls.
+        return false if @macroStack.length > 20
+
+        @macroStack << MacroStackEntry.new(macro, args, text, @nextMacroEnd)
+        true
+      end
+
+      def scan(re)
+        # We read the file line by line with gets(). If we don't have a line
+        # yet or we've reached the end of a line, we get the next one.
+        if @scanner.nil? || @scanner.eos?
+          if (@line = @stream.gets)
+            # Update activity meter about every 1024 lines.
+            Log.activity if (@stream.lineno & 0x3FF) == 0
+            # Check for DOS or Mac end of line signatures.
+            if @line[-1] == ?\r
+              # Mac: Convert CR into LF
+              @line[-1] = ?\n
+            elsif @line[-2] == ?\r
+              # DOS: Convert CR+LF into LF
+              @line = @line.chomp + "\n"
+            end
+          else
+            # We've reached the end of the current file.
+            @scanner = nil
+            # Return special EOF symbol.
+            return :scannerEOF
+          end
+          @scanner = StringScanner.new(@line)
+          @wrapped = @line[-1] == ?\n
+        end
+        return nil if (token = @scanner.scan(re)).nil?
+        #puts "#{re.to_s[0..20]}: [#{token}]"
+
+        @pos = @scanner.pos
+        while @nextMacroEnd && @nextMacroEnd < @pos
+          @macroStack.pop
+          @nextMacroEnd = @macroStack.empty? ? nil : @macroStack.last.endPos
+        end
+
+        token
+      end
+
+      def peek(n)
+        @scanner ? @scanner.peek(n) : nil
+      end
+
+      def eof?
+        @stream.eof? && @scanner.eos?
       end
 
       def dirname
         @fileName ? File.dirname(@fileName) : ''
       end
 
+      # Return the number of the currently processed line.
+      def lineNo
+        # The IO object counts the lines for us by counting the gets() calls.
+        currentLine = @stream && @scanner ? @stream.lineno : 1
+        # If we've just read the LF, we have to add 1. The LF can only be the
+        # last character of the line.
+        currentLine += 1 if @wrapped && @line && @scanner && @scanner.eos?
+        currentLine
+      end
+
+      # Return the already processed part of the current line.
+      def line
+        return '' unless @line
+
+        @line[0..(@scanner.pos - 1)]
+      end
+
     end
 
     # Specialized version of StreamHandle for operations on files.
@@ -57,53 +158,14 @@ class TaskJuggler
       def initialize(fileName)
         super()
         @fileName = fileName.dup.untaint
-        @file = fileName == '.' ? $stdin : File.new(@fileName, 'r')
-        @bytes = 0
+        @stream = fileName == '.' ? $stdin : File.new(@fileName, 'r')
         Log << "Parsing file #{@fileName} ..."
         Log.startProgressMeter("Reading file #{fileName}")
       end
 
       def close
-        @file.close unless @file == $stdin
-      end
-
-      def getc19
-        Log.activity if @bytes & 0x3FFF == 0
-
-        begin
-          # This function converts CR+LF or CR into LF on the fly.
-          if (c1 = @file.getc) == ?\r
-            @bytes += 1
-            # CR or CR/LF linebreaks
-            if (c2 = @file.getc) == ?\n
-              @bytes += 1
-              # Ok, CR, LF
-              return c2
-            else
-              # Just CR
-              @file.ungetc(c2)
-              return ?\n
-            end
-          else
-            # This is for LF linebreaks and all other characters
-            @bytes += 1
-            return c1
-          end
-        rescue
-          return nil
-        end
-      end
-
-      def getc18
-        c = getc19
-        return nil if c.nil?
-        '' << c
-      end
-
-      if RUBY_VERSION < '1.9.0'
-        alias getc getc18
-      else
-        alias getc getc19
+        @stream.close unless @stream == $stdin
+        super
       end
 
     end
@@ -113,43 +175,10 @@ class TaskJuggler
 
       def initialize(buffer)
         super()
-        @buffer = buffer
-        @length = @buffer.length_utf8
-        @pos = 0
-        Log << "Parsing buffer #{@buffer[0, 20]} ..."
-      end
-
-      def close
-        @buffer = nil
-      end
-
-      def getc18
-        return nil if @pos >= @length
-        '' << getc19
-      end
-
-      def getc19
-        return nil if @pos >= @length
-
-        # This function converts CR+LF or CR into LF on the fly.
-        if (c = @buffer[@pos]) == "\r"
-          # CR or CR+LF
-          @pos += 1 if @buffer[@pos + 1] == "\n"
-          c = "\n"
-        end
-        @pos += 1
-        c
+        @stream = StringIO.new(buffer)
+        Log << "Parsing buffer #{buffer[0, 20]} ..."
       end
 
-      if RUBY_VERSION < '1.9.0'
-        alias getc getc18
-      else
-        alias getc getc19
-      end
-
-      def fileName
-        ''
-      end
     end
 
     # Create a new instance of TextScanner. _masterFile_ must be a String that
@@ -161,33 +190,78 @@ class TaskJuggler
       # This table contains all macros that may be expanded when found in the
       # text.
       @macroTable = MacroTable.new(messageHandler)
+      # The currently processed IO object.
+      @cf = nil
       # This Array stores the currently processed nested files. It's an Array
-      # of Arrays. The nested Array consists of 3 elements, the @cf,
-      # @tokenBuffer and the @pos of the file.
+      # of Arrays. The nested Array consists of 2 elements, the IO object and
+      # the @tokenBuffer.
       @fileStack = []
-      # This Array stores the currently processed nested macros.
-      @macroStack = []
-      # In certain situation we want to ignore Macro replacement and this flag
-      # is set to true.
-      @ignoreMacros = false
+      # This flag is set if we have reached the end of a file. Since we will
+      # only know when the next new token is requested that the file is really
+      # done now, we have to use this flag.
+      @finishLastFile = false
+      # True if the scanner operates on a buffer.
       @fileNameIsBuffer = false
+      # A SourceFileInfo of the start of the currently processed token.
+      @startOfToken = nil
+      # Line number correction for error messages.
+      @lineDelta = 0
+      # Lists of regexps that describe the detectable tokens. The Arrays are
+      # grouped by mode.
+      @patternsByMode = { }
+      # The currently active scanner mode.
+      @scannerMode = nil
+      # Points to the currently active pattern set as defined by the mode.
+      @activePatterns = nil
+    end
+
+    # Add a new pattern to the scanner. _type_ is either nil for tokens that
+    # will be ignored, or some identifier that will be returned with each
+    # token of this type. _regExp_ is the RegExp that describes the token.
+    # _mode_ identifies the scanner mode where the pattern is active. If it's
+    # only a single mode, _mode_ specifies the mode directly. For multiple
+    # modes, it's an Array of modes.  _postProc_ is a method reference. This
+    # method is called after the token has been detected. The method gets the
+    # type and the matching String and returns them again in an Array.
+    def addPattern(type, regExp, mode, postProc = nil)
+      if mode.is_a?(Array)
+        mode.each do |m|
+          # The pattern is active in multiple modes
+          @patternsByMode[m] = [] unless @patternsByMode.include?(m)
+          @patternsByMode[m] << [ type, regExp, postProc ]
+        end
+      else
+        # The pattern is only active in one specific mode.
+        @patternsByMode[mode] = [] unless @patternsByMode.include?(mode)
+        @patternsByMode[mode] << [ type, regExp, postProc ]
+      end
     end
 
+    # Switch the parser to another mode. The scanner will then only detect
+    # with pattens of that _newMode_.
+    def mode=(newMode)
+      #puts "**** New mode: #{newMode}"
+      @activePatterns = @patternsByMode[newMode]
+      raise "Undefined mode #{newMode}" unless @activePatterns
+      @scannerMode = newMode
+    end
+
+
     # Start the processing. if _fileNameIsBuffer_ is true, we operate on a
     # String, else on a File.
     def open(fileNameIsBuffer = false)
       @fileNameIsBuffer = fileNameIsBuffer
       if fileNameIsBuffer
-        @fileStack = [ [ @cf = BufferStreamHandle.new(@masterFile), nil, nil ] ]
+        @fileStack = [ [ @cf = BufferStreamHandle.new(@masterFile), nil ] ]
       else
         begin
-          @fileStack = [ [ @cf = FileStreamHandle.new(@masterFile), nil, nil ] ]
+          @fileStack = [ [ @cf = FileStreamHandle.new(@masterFile), nil ] ]
         rescue StandardError
           raise TjException.new, "Cannot open file #{@masterFile}"
         end
       end
       @masterPath = @cf.dirname + '/'
-      @tokenBuffer = @pos = @lastPos = nil
+      @tokenBuffer = nil
     end
 
     # Finish processing and reset all data structures.
@@ -200,46 +274,46 @@ class TaskJuggler
       @cf = @tokenBuffer = nil
     end
 
-    # Continue processing with a new file specified by _fileName_. When this
-    # file is finished, we will continue in the old file after the location
-    # where we started with the new file.
-    def include(fileName)
-      if fileName[0] != '/'
-        if @fileStack.empty?
-          path = @masterPath == './' ? '' : @masterPath
-        else
-          pathOfCallingFile = @fileStack.last[0].dirname
-          path = pathOfCallingFile.empty? ? '' : pathOfCallingFile + '/'
-          @fileStack.last[1, 2] = [ @tokenBuffer, @pos ]
-        end
+    # Continue processing with a new file specified by _includeFileName_. When
+    # this file is finished, we will continue in the old file after the
+    # location where we started with the new file.
+    def include(includeFileName, sfi)
+      if includeFileName[0] != '/'
+        pathOfCallingFile = @fileStack.last[0].dirname
+        path = pathOfCallingFile.empty? ? '' : pathOfCallingFile + '/'
         # If the included file is not an absolute name, we interpret the file
         # name relative to the including file.
-        fileName = path + fileName
+        includeFileName = path + includeFileName
       end
 
-      @tokenBuffer = nil
-
-      # Check the current file stack to find recusions.
-      if @pos && fileName == @pos.fileName
-        error('include_recursion', "Recursive inclusion of #{fileName} detected")
-      else
-        @fileStack.each do |entry|
-          if fileName == entry[0].fileName
-            error('include_recursion', "Recursive inclusion of #{fileName} " +
-                  "detected")
-          end
+      # Try to dectect recursive inclusions. This will not work if files are
+      # accessed via filesystem links.
+      @fileStack.each do |entry|
+        if includeFileName == entry[0].fileName
+          error('include_recursion',
+                "Recursive inclusion of #{includeFileName} detected", nil, sfi)
         end
       end
+
+      # Save @tokenBuffer in the record of the parent file.
+      @fileStack.last[1] = @tokenBuffer unless @fileStack.empty?
+      @tokenBuffer = nil
+      @finishLastFile = false
+
+      # Open the new file and push the handle on the @fileStack.
       begin
-        @fileStack << [ (@cf = FileStreamHandle.new(fileName)), nil, nil ]
+        @fileStack << [ (@cf = FileStreamHandle.new(includeFileName)), nil, ]
+        Log << "Parsing file #{includeFileName}"
       rescue StandardError
-        error('bad_include', "Cannot open include file #{fileName}")
+        error('bad_include', "Cannot open include file #{includeFileName}",
+              nil, sfi)
       end
     end
 
     # Return SourceFileInfo for the current processing prosition.
     def sourceFileInfo
-      @pos ? @pos.dup : SourceFileInfo.new(fileName, lineNo, columnNo)
+      @cf ? SourceFileInfo.new(fileName, @cf.lineNo - @lineDelta, 0) :
+            SourceFileInfo.new(@masterFile, 0, 0)
     end
 
     # Return the name of the currently processed file. If we are working on a
@@ -253,94 +327,97 @@ class TaskJuggler
     end
 
     def columnNo # :nodoc:
-      @cf ? @cf.columnNo : 0
+      0
     end
 
     def line # :nodoc:
       @cf ? @cf.line : 0
     end
 
-    # Scan for the next token in the input stream and return it. The result will
-    # be an Array of the form [ TokenType, TokenValue ].
+    # Return the next token from the input stream. The result is an Array with
+    # 3 entries: the token type, the token String and the SourceFileInfo where
+    # the token started.
     def nextToken
       # If we have a pushed-back token, return that first.
       unless @tokenBuffer.nil?
         res = @tokenBuffer
         @tokenBuffer = nil
-        @pos = @tokenBufferPos
         return res
       end
 
+      if @finishLastFile
+        # The previously processed file has now really been processed to
+        # completion. Close it and remove the corresponding entry from the
+        # @fileStack.
+        @finishLastFile = false
+        #Log << "Completed file #{@cf.fileName}"
+        @cf.close
+        @fileStack.pop
+
+        if @fileStack.empty?
+          # We are done with the top-level file now.
+          @cf = @tokenBuffer = nil
+          @finishLastFile = true
+          return [ '.', '<END>', @startOfToken ]
+        else
+          # Continue parsing the file that included the current file.
+          @cf, tokenBuffer = @fileStack.last
+          Log << "Parsing file #{@cf.fileName} ..."
+          # If we have a left over token from previously processing this file,
+          # return it now.
+          return tokenBuffer if tokenBuffer
+        end
+      end
+
       # Start processing characters from the input.
-      @startOfToken = SourceFileInfo.new(fileName, lineNo, columnNo)
-      token = [ '.', '<END>' ]
-      while c = nextChar
-        case c
-        when ' ', "\n", "\t"
-          if (tok = readBlanks(c))
-            token = tok
-            break
+      @startOfToken = sourceFileInfo
+      loop do
+        match = nil
+        begin
+          @activePatterns.each do |type, re, postProc|
+            if (match = @cf.scan(re))
+              if match == :scannerEOF
+                # We've found the end of an input file. Return a special token
+                # that describes the end of a file.
+                @finishLastFile = true
+                return [ '.', '<END>', @startOfToken ]
+              end
+
+              raise "#{re} matches empty string" if match.empty?
+              # If we have a post processing method, call it now. It may modify
+              # the type or the found token String.
+              type, match = postProc.call(type, match) if postProc
+
+              break if type.nil? # Ignore certain tokens with nil type.
+
+              return [ type, match, @startOfToken ]
+            end
           end
-          @startOfToken = SourceFileInfo.new(fileName, lineNo, columnNo)
-        when '#'
-          skipComment
-          @startOfToken = SourceFileInfo.new(fileName, lineNo, columnNo)
-        when '/'
-          skipCPlusPlusComments
-          @startOfToken = SourceFileInfo.new(fileName, lineNo, columnNo)
-          when '0'..'9'
-          token = readNumber(c)
-          break
-        when "'"
-          token = readString(c)
-          break
-        when '"'
-          token = readString(c)
-          break
-        when '-'
-          token = handleDash
-          break
-        when '!'
-          if (c = nextChar) == '='
-            token = [ 'LITERAL', '!=' ]
+        rescue
+          error('scan_encoding_error', $!.to_s)
+        end
+
+        if match.nil?
+          if @cf.eof?
+            error('unexpected_eof',
+                  "Unexpected end of file found")
           else
-            returnChar(c)
-            token = [ 'LITERAL', '!' ]
+            error('no_token_match',
+                  "Unexpected characters found: '#{@cf.peek(10)}...'")
           end
-          break
-        when 'a'..'z', 'A'..'Z', '_'
-          token = readId(c)
-          break
-        when '<', '>', '='
-          token = readOperator(c)
-          break
-        when '['
-          token = readMacro
-          break
-        when nil
-          # We've reached an end of file or buffer
-          break
-        else
-          str = ""
-          str << c
-          token = [ 'LITERAL', str ]
-          break
         end
       end
-      @pos = @startOfToken
-      return token
     end
 
     # Return a token to retrieve it with the next nextToken() call again. Only 1
     # token can be returned before the next nextToken() call.
     def returnToken(token)
+      #Log << "-> Returning Token: [#{token[0]}][#{token[1]}]"
       unless @tokenBuffer.nil?
         $stderr.puts @tokenBuffer
         raise "Fatal Error: Cannot return more than 1 token in a row"
       end
       @tokenBuffer = token
-      @tokenBufferPos = @pos
-      @pos = @lastPos
     end
 
     # Add a Macro to the macro translation table.
@@ -353,548 +430,54 @@ class TaskJuggler
       @macroTable.include?(name)
     end
 
-    def expandMacro(args)
+    # Expand a macro and inject it into the input stream. _prefix_ is any
+    # string that was found right before the macro call. We have to inject it
+    # before the expanded macro. _args_ is an Array of Strings. The first is
+    # the macro name, the rest are the parameters.
+    def expandMacro(prefix, args)
+      # Get the expanded macro from the @macroTable.
       macro, text = @macroTable.resolve(args, sourceFileInfo)
+      # If the expanded macro is empty, we can ignore it.
       return if text == ''
 
-      if @macroStack.length > 20
+      unless @cf.injectMacro(macro, args, prefix + text)
         error('macro_stack_overflow', "Too many nested macro calls.")
       end
-      @macroStack << [ macro, args ]
-      # Mark end of macro with a 0 element
-      @cf.charBuffer << 0
-      text.reverse.each_utf8_char do |c|
-        @cf.charBuffer << c
-      end
-      @cf.line = ''
     end
 
     # Call this function to report any errors related to the parsed input.
-    def error(id, text, property = nil)
-      message('error', id, text, property)
+    def error(id, text, property = nil, sfi = nil)
+      message('error', id, text, property, sfi)
     end
 
-    def warning(id, text, property = nil)
-      message('warning', id, text, property)
+    def warning(id, text, property = nil, sfi = nil)
+      message('warning', id, text, property, sfi)
     end
 
-    def message(type, id, text, property)
+    def message(type, id, text, property, sfi)
       unless text.empty?
         message = Message.new(id, type, text + "\n" + line.to_s,
-                              property, nil, sourceFileInfo)
+                              property, nil, sfi || sourceFileInfo)
         @messageHandler.send(message)
 
-        until @macroStack.empty?
-          macro, args = @macroStack.pop
-          args.collect! { |a| '"' + a + '"' }
-          message = Message.new('macro_stack', 'info',
-                                "   #{macro.name} #{args.join(' ')}", nil, nil,
-                                macro.sourceFileInfo)
+        if @cf && !@cf.macroStack.empty?
+          message = Message.new('macro_stack', 'info', 'Macro call history:')
           @messageHandler.send(message)
-        end
-      end
-
-      # An empty strings signals an already reported error
-      raise TjException.new, '' if type == 'error'
-    end
 
-  private
-
-    # This function is called by the scanner to get the next character. It
-    # features a FIFO buffer that can hold any amount of returned characters.
-    # When it has reached the end of the master file it returns nil.
-    def nextChar
-      # We've started to find the next token. @pos no longer marks the
-      # position of the current token. Since we also store the EOF position in
-      # @pos, don't reset it if we have processed all files completely.
-      if @pos && @cf
-        @lastPos = @pos
-        @pos = nil
-      end
-
-      if (c = nextCharI) == '$' && !@ignoreMacros
-        # Double $ are reduced to a single $.
-        return c if (c = nextCharI) == '$'
-
-        # Macros start with $( or ${. All other $. are ignored.
-        if c != '(' && c != '{'
-           returnChar(c)
-           return '$'
-        end
-
-        @ignoreMacros = true
-        returnChar(c)
-        macroParser = MacroParser.new(self, @messageHandler)
-        begin
-          macroParser.parse('macroCall')
-        rescue TjException
-        end
-        @ignoreMacros = false
-        return nextCharI
-      else
-        return c
-      end
-    end
-
-    def nextCharI
-      # This can only happen when a previous call already returned nil.
-      return nil if @cf.nil?
-
-      c = nil
-      # If there are characters in the return buffer process them first.
-      # Otherwise get next character from input stream.
-      unless @cf.charBuffer.empty?
-        c = @cf.charBuffer.pop
-        @cf.lineNo -= 1 if c == "\n" && !@macroStack.empty?
-        while !@cf.charBuffer.empty? && @cf.charBuffer[-1] == 0
-          @cf.charBuffer.pop
-          @macroStack.pop
-        end
-      else
-        # If EOF has been reached, try the parent file until even the master
-        # file has been processed completely.
-        if (c = @cf.getc).nil?
-          # Safe current position so an EOF related error can be properly
-          # reported.
-          @pos = sourceFileInfo
-
-          @cf.close
-          @fileStack.pop
-          if @fileStack.empty?
-            # We are done with the top-level file now.
-            @cf = @tokenBuffer = nil
-          else
-            @cf, @tokenBuffer, @lastPos = @fileStack.last
-            Log << "Parsing file #{@cf.fileName} ..."
-            # We have been called by nextToken() already, so we can't just
-            # restore @tokenBuffer and be done. We need to feed the token text
-            # back into the charBuffer and return the first character.
-            if @tokenBuffer
-              @tokenBuffer[1].reverse.each_utf8_char do |ch|
-                @cf.charBuffer.push(ch)
-              end
-              @tokenBuffer = nil
-            end
+          @cf.macroStack.reverse_each do |entry|
+            macro = entry.macro
+            args = entry.args[1..-1]
+            args.collect! { |a| '"' + a + '"' }
+            message = Message.new('macro_stack', 'info',
+                                  "  ${#{macro.name} #{args.join(' ')}}",
+                                  nil, nil, macro.sourceFileInfo)
+            @messageHandler.send(message)
           end
-          return nil
         end
       end
-      unless c.nil?
-        @cf.lineNo += 1 if c == "\n"
-        @cf.line = "" if @cf.line[-1] == ?\n
-        @cf.line << c
-      end
-      c
-    end
-
-    def returnChar(c)
-      return if @cf.nil?
 
-      @cf.line.chop! if c
-      @cf.charBuffer << c
-      @cf.lineNo -= 1 if c == "\n" && @macroStack.empty?
-    end
-
-    def skipComment
-      # Read all characters until line or file end is found
-      @ignoreMacros = true
-      while (c = nextChar) && c != "\n"
-      end
-      @ignoreMacros = false
-      returnChar(c)
-    end
-
-    def skipCPlusPlusComments
-      if (c = nextChar) == '*'
-        # /* */ style multi-line comment
-        @ignoreMacros = true
-        begin
-          while (c = nextChar) != '*'
-          end
-        end until (c = nextChar) == '/'
-        @ignoreMacros = false
-      elsif c == '/'
-        # // style single line comment
-        skipComment
-      else
-        error('bad_comment', "'/' or '*' expected after start of comment")
-      end
-    end
-
-    def handleDash
-      if (c1 = nextChar) == '8'
-        if (c2 = nextChar) == '<'
-          if (c3 = nextChar) == '-'
-            return readIndentedString
-          else
-            returnChar(c3)
-            returnChar(c2)
-            returnChar(c1)
-          end
-        else
-          returnChar(c2)
-          returnChar(c1)
-        end
-      else
-        returnChar(c1)
-      end
-      return [ 'LITERAL', ' - ']
-    end
-
-    def readBlanks(c)
-      loop do
-        if c == ' '
-          if (c2 = nextChar) == '-'
-            # Special case for the dash between period dates. It must be
-            # surrounded by blanks.
-            if (c3 = nextChar) == ' '
-              return [ 'LITERAL', ' - ']
-            end
-            returnChar(c3)
-          end
-          returnChar(c2)
-        elsif c != "\n" && c != "\t"
-          returnChar(c)
-          return nil
-        end
-        c = nextChar
-      end
-    end
-
-    def readNumber(c)
-      token = ""
-      token << c
-      while ('0'..'9') === (c = nextChar)
-        token << c
-      end
-      if c == '-'
-        return readDate(token)
-      elsif c == '.'
-        frac = readDigits
-
-        return [ 'FLOAT', token.to_f + frac.to_f / (10.0 ** frac.length) ]
-      elsif c == ':'
-        hours = token.to_i
-        mins = readDigits.to_i
-        if hours < 0 || hours > 24
-          raise TjException.new, "Hour must be between 0 and 23"
-        end
-        if mins < 0 || mins > 59
-          raise TjException.new, "Minutes must be between 0 and 59"
-        end
-        if hours == 24 && mins != 0
-          raise TjException.new, "Time may not be larger than 24:00"
-        end
-
-        # Return time as seconds of day since midnight.
-        return [ 'TIME', hours * 60 * 60 + mins * 60 ]
-      else
-        returnChar(c)
-      end
-
-      [ 'INTEGER', token.to_i ]
-    end
-
-    def readDate(token)
-      year = token.to_i
-      if year < 1970 || year > 2030
-        raise TjException.new, "Year must be between 1970 and 2030"
-      end
-
-      month = readDigits.to_i
-      if month < 1 || month > 12
-        raise TjException.new, "Month must be between 1 and 12"
-      end
-      if nextChar != '-'
-        raise TjException.new, "Corrupted date"
-      end
-
-      day = readDigits.to_i
-      if day < 1 || day > 31
-        raise TjException.new, "Day must be between 1 and 31"
-      end
-
-      if (c = nextChar) != '-'
-        returnChar(c)
-        return [ 'DATE', TjTime.local(year, month, day) ]
-      end
-
-      hour = readDigits.to_i
-      if hour < 0 || hour > 23
-        raise TjException.new, "Hour must be between 0 and 23"
-      end
-
-      if nextChar != ':'
-        raise TjException.new, "Corrupted time. ':' expected."
-      end
-
-      minutes = readDigits.to_i
-      if minutes < 0 || minutes > 59
-        raise TjException.new, "Minutes must be between 0 and 59"
-      end
-
-      if (c = nextChar) == ':'
-        seconds = readDigits.to_i
-        if seconds < 0 || seconds > 59
-          raise TjException.new, "Seconds must be between 0 and 59"
-        end
-      else
-        seconds = 0
-        returnChar(c)
-      end
-
-      if (c = nextChar) != '-'
-        returnChar(c)
-        return [ 'DATE', TjTime.local(year, month, day, hour, minutes, seconds) ]
-      end
-
-      if (c = nextChar) == '-'
-        delta = 1
-      elsif c == '+'
-        delta = -1
-      else
-        # An actual time zone name
-        tz = readId(c)[1]
-        oldTz = ENV['TZ']
-        ENV['TZ'] = tz
-        timeVal = TjTime.local(year, month, day, hour, minutes, seconds)
-        ENV['TZ'] = oldTz
-        if timeVal.to_a[9] != tz
-          raise TjException.new, "Unknown time zone #{tz}"
-        end
-        return [ 'DATE', timeVal ]
-      end
-
-      utcDiff = readDigits
-      utcHour = utcDiff[0, 2].to_i
-      if utcHour < 0 || utcHour > 23
-        raise TjException.new, "Hour must be between 0 and 23"
-      end
-      utcMin = utcDiff[2, 2].to_i
-      if utcMin < 0 || utcMin > 59
-        raise TjException.new, "Minutes must be between 0 and 59"
-      end
-
-      [ 'DATE', TjTime.gm(year, month, day, hour, minutes, seconds) +
-                delta * ((utcHour * 3600) + utcMin * 60) ]
-    end
-
-    def readString(terminator)
-      token = ""
-      while (c = nextChar) && c != terminator
-        if c == "\\"
-          # Terminators can be used as regular characters when prefixed by a \.
-          if (c = nextChar) && c != terminator
-            # \ followed by non-terminator. Just add both.
-            token << "\\"
-          end
-        end
-        token << c
-      end
-
-      [ 'STRING', token ]
-    end
-
-    def readIndentedString
-      state = 0
-      indent = ''
-      token = ''
-      while true
-        case state
-        when 0 # Determining indent
-          # Skip trailing spaces and tabs.
-          while (c = nextChar) == ' ' || c == "\t" do
-            # empty on purpose
-          end
-          if c != "\n"
-            error('junk_after_cut',
-                  'The cut mark -8<- must be immediately followed by a ' +
-                  'line break.')
-          end
-          while (c = nextChar) == ' ' || c == "\t"
-            indent << c
-          end
-          @startOfToken = SourceFileInfo.new(fileName, lineNo, columnNo)
-          returnChar(c)
-          state = 1
-        when 1 # reading '-' or first content line character
-          if (c = nextChar) == '-'
-            state = 3
-          elsif c.nil?
-            errorEOF(1, token)
-          elsif c == "\n"
-            token << c
-            state = 6
-          else
-            token << c
-            state = 2
-          end
-        when 2 # reading content line
-          # The '->8-' is only valid if no other content preceded it on this
-          # line.
-          onlyBlanks = true
-          while (c = nextChar) != "\n" && !(c == '-' && onlyBlanks)
-            onlyBlanks = false if c != ' ' && c != "\t"
-            errorEOF(2, token) if c.nil?
-            token << c
-          end
-          if c == '-'
-            # we may have found the start of '->8-'
-            state = 3
-          else
-            token << c
-            state = 6
-          end
-        when 3 # reading '>' of '->8-'
-          if (c = nextChar) == '>'
-            state = 4
-          else
-            errorEOF(3, token) if c.nil?
-            token << '-'
-            token << c
-            state = 2
-          end
-        when 4 # reading '8' of '->8-'
-          if (c = nextChar) == '8'
-            state = 5
-          else
-            errorEOF(4, token) if c.nil?
-            token << c
-            state = 2
-          end
-        when 5 # reading '-' of '->8-'
-          if (c = nextChar) == '-'
-            return [ 'STRING', token ]
-          else
-            errorEOF(5, token) if c.nil?
-            token << c
-            state = 2
-          end
-        when 6 # reading indentation
-          state = 1
-          indent.each_utf8_char do |ci|
-            if ci != (c = nextChar)
-              if c == '-'
-                state = 3
-                break
-              elsif c == "\n"
-                returnChar(c)
-                break
-              else
-                warning('bad_indent',
-                        "Not all lines of string have same indentation. " +
-                        "The first line of the string determines the " +
-                        "indentation for all subsequent lines of the same " +
-                        "string.")
-                token << c
-                state = 2
-                break
-              end
-            end
-          end
-        else
-          raise "State machine error"
-        end
-      end
-    end
-
-    def readId(c)
-      token = ""
-      token << c
-      while (c = nextChar) &&
-            (('a'..'z') === c || ('A'..'Z') === c || ('0'..'9')  === c ||
-             c == '_')
-        token << c
-      end
-      if c == ':'
-        return [ 'ID_WITH_COLON', token ]
-      elsif  c == '.'
-        token << c
-        loop do
-          token += readIdentifier
-          break if (c = nextChar) != '.'
-          token += '.'
-        end
-        returnChar c
-
-        return [ 'ABSOLUTE_ID', token ]
-      else
-        returnChar c
-        return [ 'ID', token ]
-      end
-    end
-
-    def readMacro
-      # We can deal with ']' inside of the macro as long as each ']' is
-      # preceeded by a corresponding '['.
-      token = ''
-      bracketLevel = 1
-      while bracketLevel > 0
-        case (c = nextCharI)
-        when nil
-          error('unterminated_macro', "Unterminated macro #{token}")
-        when '['
-          bracketLevel += 1
-        when ']'
-          bracketLevel -= 1
-        end
-        token << c unless bracketLevel == 0
-      end
-      return [ 'MACRO', token ]
-    end
-
-    # Read operators of logical expressions.
-    def readOperator(c)
-      case c
-      when '='
-        return [ 'LITERAL', '=' ]
-      when '>'
-        return [ 'LITERAL', '>=' ] if (c = nextChar) == '='
-        returnChar(c)
-        return [ 'LITERAL', '>' ]
-      when '<'
-        return [ 'LITERAL', '<=' ] if (c = nextChar) == '='
-        returnChar(c)
-        return [ 'LITERAL', '<' ]
-      else
-        raise "Unsupported operator #{c}"
-      end
-    end
-
-    # Read only decimal digits and return the result als Fixnum.
-    def readDigits
-      token = ""
-      while ('0'..'9') === (c = nextChar)
-        token << c
-      end
-      # Make sure that we have read at least one digit.
-      if token == ""
-        raise TjException.new, "Digit (0 - 9) expected"
-      end
-      # Push back the non-digit that terminated the digits.
-      returnChar(c)
-      token
-    end
-
-    def readIdentifier(noDigit = true)
-      token = ""
-      while (c = nextChar) &&
-            (('a'..'z') === c || ('A'..'Z') === c ||
-             (!noDigit && (('0'..'9')  === c)) || c == '_')
-        token << c
-        noDigit = false
-      end
-      returnChar(c)
-      if token == ""
-        raise TjException.new, "Identifier expected"
-      end
-      token
-    end
-
-    def errorEOF(no, token)
-      error("eof_in_istring#{no}",
-            "Unexpected end of file in string '#{token[0,20]}...'" +
-            "starting in line #{@startOfToken.lineNo}.")
+      # An empty strings signals an already reported error
+      raise TjException.new, '' if type == 'error'
     end
 
   end