xiacc 0.1

data/xiacc-0.1/lib/nist/xiacc/InstanceManagerFromSpec.rb

@@ -0,0 +1,70 @@
+module XI
+
+  require 'nist/xiacc/InstanceManager'
+
+  # This class represents named objects which are created
+  # from a named spec through an intermediate generator object (not named).
+  #
+  # Subclasses must override these messages:
+  #          class side:
+  #             specClass
+  #             generatorClass
+  #          instance side:
+  #             generator: (Initialize the instance from the generator)
+  #
+  #
+  # The generator must support this protocol:
+  #          class side:
+  #             spec:  (Returns a new instance initialized from the spec)
+  #          instance side:
+  #             name
+  #             whatever is required by the above generator: instance side method
+
+  class InstanceManagerFromSpec < InstanceManager
+    
+    #instance creation
+    def newNamed( aSymbol )
+      newNamedWithClass( aSymbol, generatorClass )
+    end
+    
+    def InstanceManagerFromSpec.newNamedWithSpecNameAndGeneratorClass( instanceName, specName, generatorClass )
+      spec = specNamed( instanceName )
+      return InstanceManagerFromSpec.newNamedFromGenerator( instanceName, eval( generatorClass ).spec( spec ) )
+    end
+
+    def InstanceManagerFromSpec.newNamed( aSymbol )
+      return InstanceManagerFromSpec.newNamedWithGeneratorClass( aSymbol, generatorClass )
+    end
+
+    def InstanceManagerFromSpec.fromGenerator( generator, className )
+      return InstanceManagerFromSpec.newNamedFromGenerator( generator.name, generator, className )
+    end
+
+    def InstanceManagerFromSpec.generatorClass
+      error( "Oops, this shouldn't be called, subclass responsibility\n" )
+    end
+
+    def InstanceManagerFromSpec.newNamedFromGenerator( aSymbol, aGenerator, className )
+      imfs = InstanceManagerFromSpec.defaultNamedWithClass( aSymbol, className )
+      imfs.setGenerator( aGenerator )
+      imfs
+    end
+    
+    def InstanceManagerFromSpec.newNamedGeneratorClass( instanceAndSpecName, generatorClass )
+      InstanceManagerFromSpec.newNamedWithSpecNameAndGeneratorClass( instanceAndSpecName, instanceAndSpecName, generatorClass )
+    end
+
+    def InstanceManagerFromSpec.specClass
+      error( "Oops, this shouldn't be called, subclass responsibility\n" )
+    end
+
+    def InstanceManagerFromSpec.specNamed( instanceName )
+      specClass.named( instanceName )
+    end
+
+    def setGenerator( aGenerator )
+      error( "Subclass responsibility" )
+    end
+
+  end
+end

data/xiacc-0.1/lib/nist/xiacc/ItemSet.rb

@@ -0,0 +1,43 @@
+module XI
+  require 'nist/xiacc/UnHashedSet'
+
+  class ItemSet < UnHashedSet
+
+    def initialize
+      super
+    end
+
+    def ItemSet.fromArray( arr )
+      is = ItemSet.new
+      arr.each { |val| is.push( val ) }
+      is
+    end
+    
+    def acceptsProduction?( aProduction )
+      #answer = select { |item| item.isAtEnd? and item.production.equal?( aProduction ) and item.lookahead.isEndMarker? }
+      each { |item| return true if ( item.isAtEnd? and item.production.eql?( aProduction ) and item.lookahead.isEndMarker? ) }
+      return false
+    end
+
+    def containsProduction?( aProduction )
+      #print "looking for production: #{aProduction.to_s} my size is: #{size}\n"
+      each { |item| return true if item.production.eql?( aProduction ) }
+      return false
+    end
+
+    def endItemsWithLookahead( aTerminal )
+      ( select { |item| item.isAtEnd? and item.lookahead.eql?( aTerminal ) } ).to_itemSet
+    end
+
+    def endItemWithLookahead( aTerminal )
+      ( select { |item| item.isAtEnd? and item.lookahead.equal?( aTerminal ) } ).to_itemSet
+    end
+
+    def includesNextSymbol?( symbol )
+      #print "in includeNextSymbol? : \n"
+      each { |item| return true if item.hasNextSymbol?( symbol ) == true }
+      return false
+    end
+
+  end
+end

data/xiacc-0.1/lib/nist/xiacc/KeyedCollection.rb

@@ -0,0 +1,12 @@
+require 'nist/xiacc/Collection'
+require 'nist/xiacc/OrderedCollection'
+
+class KeyedCollection < Array
+
+  def associations
+    answer = Array.new
+    
+    each_pair { |key, value| answer[ key ] = value }
+  end
+
+end

data/xiacc-0.1/lib/nist/xiacc/LRParserGenerator.rb

@@ -0,0 +1,127 @@
+module XI
+
+  require 'nist/xiacc/CCObjectAddon'
+
+  class LRParserGenerator
+
+    attr :language, true
+    attr :stateKeyedDictionary, true
+    attr :initialState, true
+    attr :warnings, true
+    
+    def initialize 
+      super
+      @warnings = Array.new
+      @stateKeyedDictionary = Hash.new
+    end
+
+    def LRParserGenerator.language( aLanguage )
+      ( lrpg = LRParserGenerator.new ).language = aLanguage
+      lrpg
+    end
+
+    def LRParserGenerator.spec( aLanguage )
+      ( @language = aLanguage ).generateStates
+      @language
+    end
+
+    def createStateForItems( parserItems )
+      newState = stateClass.alternativeInitialize( @stateKeyedDictionary.size + 1, parserItems )
+      @stateKeyedDictionary[ newState ] = parserItems
+      newState
+    end
+
+    def first( aGrammarStringOrGrammarSymbol )
+      @language.unshift = aGrammarStringOrGrammarSymbol
+    end
+
+    def generateInitialState
+      seedItem = itemClass.alternativeInitialize( @language.startProduction, 0, EndMarker.default )
+      @initialState = createStateForItems( closureOfItemSet( [ seedItem ] ) )
+      #print "initialState is: #{initialState.to_s}\n"
+      @initialState
+      #exit
+    end
+
+    def itemClass
+      error( "A subclass should have defined this: LRParserGenerator#itemClass" )
+    end
+
+    def itemsForState( aParserState )
+      @stateKeyedDictionary[ aParserState ].to_itemSet
+    end
+    
+    def name
+      @language.name
+    end
+
+    def productionDictionary
+      @language.productionDictionary
+    end
+
+    def productions
+      @language.productions
+    end
+
+    def productionsForNonTerminal( aNonTerminal )
+      @language.productionsForNonTerminal( aNonTerminal )
+    end
+
+    def stateClass
+      error( "A subclass should have defined this: LRParserGenerator#stateClass" )
+    end
+
+    def stateForItems( parserItems )
+      return testAbsenseForStateForItems( parserItems ) { createStateForItems( parserItems ) }
+    end
+    
+    def testAbsenseForStateForItems( parserItems, &block )
+      #print "value has type: "
+      #@stateKeyedDictionary.each_pair do
+      #  |key,value| 
+      #  value.each { |val| print val.class.to_s + "\n" }
+      #end
+
+      #print "stateKeyedDictionary: #{@stateKeyedDictionary.to_a.join( "   and   " )}\n"
+      #print "hash #'s: "
+      #@stateKeyedDictionary.each_pair { |key,value| print "KEY: #{key.hash}" }
+      #@stateKeyedDictionary.each_pair { |key,value| print "value: #{value.hash}" }
+      #opposite = @stateKeyedDictionary.invert
+      #return block.call if opposite[ parserItems ] == nil
+      #opposite[ parserItems ]
+      found = false
+      retval = nil
+      @stateKeyedDictionary.each_pair do
+	|key,value|
+	if value.eql?( parserItems )
+	  found = true
+	  retval = key
+	  break
+	end
+      end
+      return block.call if !found
+      return retval
+    end
+
+    def generateStates
+      error( "Subclass responsibility" )
+    end
+
+    def printStatesOn( stream )
+      #print "size: #{@stateKeyedDictionary.keys.size}\n"
+      stream << "---------------------------------\n" 
+      @stateKeyedDictionary.each_pair do
+	|key,value|
+	#print "matey.. state has class: #{key.class.to_s}\n"
+	stream << "State: #{key.to_s} "
+	#print "Aye\n"
+	#$defout.flush
+	stream << "and ItemSet: #{value.to_s}\n----\n"
+      end
+      stream << "---------------------------------\n" 
+    end
+
+    
+  end
+
+end

data/xiacc-0.1/lib/nist/xiacc/LRParserItem.rb

@@ -0,0 +1,118 @@
+module XI
+
+  require 'nist/xiacc/CCObjectAddon'
+  require 'nist/xiacc/GrammarString'
+
+  class LRParserItem
+
+    attr :production, true
+    attr :index, true
+
+    def initialize
+      super
+      @index = -1
+    end
+
+    def LRParserItem.alternativeInitialize( aProduction, positiveInteger )
+      lrpi = LRParserItem.new
+      lrpi.production = aProduction
+      lrpi.index = positiveInteger
+    end
+    
+    # comparison
+    def ==( anotherInstance )
+      return false if self.class.equal?( anotherInstance.class ) == false
+      return ( @production.equal?( anotherInstance.production ) and @index == anotherInstance.index )
+    end
+
+    def eql?( another )
+      return true if self.hash == another.hash
+      return false
+    end
+    
+    def hash
+      #print "lr_parser_item\#hash, production is: #{@production} and production's hash: #{@production.hash} and index's: #{@index.hash}\n"
+      return @production.hash + @index.hash
+    end
+
+    # operation
+    def advance
+      @index = @index + 1
+    end
+
+    def printOn( aStream )
+      #super( aStream )
+      #aStream << "  "
+      @production.printHeadOn( aStream )
+      @expansion = @production.expansion
+      #@index.times do
+      #  |idx|
+      #  @expansion[ idx ].printNameOn( aStream )
+      #end
+      #aStream << ". "
+      #s = @expansion.size - index
+      #s = s - 1
+      #s.times do 
+      #  |idx|
+      #  pos = index + idx
+      #  @expansion[ pos ].printNameOn( aStream )
+      #end
+      dotDone = false
+      @expansion.size.times do 
+	|idx|
+	if idx == @index
+	  aStream << ". "
+	  dotDone = true
+	end
+	@expansion[ idx ].printNameOn( aStream )
+	aStream << " "
+      end
+      if dotDone == false
+	aStream << "."
+      end
+
+    end
+
+    def isAtEnd?
+      return @index >= @production.expansion.size
+    end
+    
+    def isAtStart?
+      @index == 0
+    end
+    
+    def nextGrammarSymbol
+      #print "its class: #{@production.expansion[ @index ].class.to_s}\n"
+      # XXXXXXXX danger, will robinson, danger.... changed from @index + 1
+      @production.expansion[ @index ]
+    end
+
+    def nextNonTerminal
+      if isAtEnd?
+	return nil
+      end
+      n = nextGrammarSymbol
+      # XXXXXXXXXX danger, will robinson, danger.... changed from Terminal to NonTerminal
+      if n.class == NonTerminal 
+	return n
+      end
+      nil
+    end
+
+    def remainingGrammarSymbols
+      if isAtEnd?
+	return GrammarString.new
+      end
+      pe = @production.expansion
+      # XXXXXXXXX this may not be good/... may have to change back to @index + 1
+      return pe[ @index .. pe.size + 1 ]
+    end
+
+    # not categorized
+    def hasNextSymbol?( symbol )
+      return false if isAtEnd? == true
+      return symbol == nextGrammarSymbol
+    end
+
+  end
+end

data/xiacc-0.1/lib/nist/xiacc/Language.rb

@@ -0,0 +1,577 @@
+module XI
+
+  require 'nist/xiacc/InstanceManager'
+  require 'nist/xiacc/ParseTreeNode'
+  require 'nist/xiacc/NonTerminal'
+  require 'nist/xiacc/EndMarker'
+  require 'nist/xiacc/Epsilon'
+  require 'nist/xiacc/CCArrayAddon'
+  require 'nist/xiacc/CCStringAddon'
+  require 'nist/xiacc/CCHashAddon'
+  require 'nist/xiacc/GrammarString'
+  require 'nist/xiacc/ItemSet'
+  require 'nist/xiacc/TerminalDescription'
+  require 'nist/xiacc/CCAction'
+  require 'nist/xiacc/Shortcut'
+
+  class Language < InstanceManager
+
+    attr :terminalDict, true 
+    attr :nonTerminalDict, true
+    attr :terminals, true
+    attr :nonTerminals, true
+    attr :productionDict, true
+    attr :startProduction, true
+    attr :firstDict, true
+    attr :followsDict, true
+    attr :terminalDescriptions, true
+
+    def initialize
+      @terminalDict = Hash.new
+      @terminalDescriptions = Hash.new
+      @nonTerminalDict = Hash.new
+      @productionDict = Hash.new
+      @firstDict = Hash.new
+      @followsDict = Hash.new
+
+      @terminalDict[ "epsilon" ] = Epsilon.default
+      @terminalDict[ "endMarker" ] = EndMarker.default
+    end
+
+    # tests
+    def Language.page195
+      answer = Language.new
+      answer.setTerminalCharacters( "a b c d e" )
+      answer.setNonTerminalNames( "S A B" )
+      answer.addExpansionList( "S -> a A B e" )
+      answer.addExpansionList( "A -> A b c" )
+      answer.addExpansionList( "A -> b" )
+      answer.addExpansionList( "B -> d" )
+      answer
+    end
+    
+    def Language.page231
+      answer = Language.new
+      answer.setTerminalCharacters( "c d" )
+      answer.setNonTerminalNames( "S C" )
+      answer.addExpansionList( "S -> C C" )
+      answer.addExpansionList( "C -> c C" )
+      answer.addExpansionList( "C -> d" )
+      answer
+    end
+
+    def Language.page222
+      answer = Language.new
+      answer.setTerminalCharacters( "( ) * + id" )
+      answer.setNonTerminalNames( "E T F" )
+      answer.addExpansionList( "E -> E + T" )
+      answer.addExpansionList( "E -> T" )
+      answer.addExpansionList( "T -> T * F" )
+      answer.addExpansionList( "T -> F" )
+      answer.addExpansionList( "F -> ( E )" )
+      answer.addExpansionList( "F -> id" )
+      answer
+    end
+
+    def Language.wth
+      answer = Language.new
+      answer.setTerminalCharacters( "a b c d" )
+      answer.setNonTerminalNames( "E A" )
+      answer.addExpansionList( "E -> A b" )
+      answer.addExpansionList( "A -> a" )
+      answer
+    end
+
+    def Language.pageX
+      answer = Language.new
+      answer.addTerminalDescription( "digi", Regexp.compile( /[[:digit:]]/ ) )
+      answer.addTerminalDescription( "o", Regexp.compile( /[abc]/ ) )
+      answer.setNonTerminalNames( "E A" )
+      answer.addExpansionList( "E -> A" )
+      answer.addExpansionList( "A -> digi A" )
+      answer.addExpansionList( "A -> o A" )
+      answer.addExpansionList( "A -> digi" )
+      answer.addExpansionList( "A -> o" )
+      answer
+    end
+
+    def Language.pageXY
+      answer = Language.new
+      answer.addTerminalDescription( "digi", Regexp.compile( /[[:digit:]]/ ) )
+      answer.addTerminalDescription( "o", Regexp.compile( /[abc]/ ) )
+      answer.setNonTerminalNames( "E A" )
+      answer.addExpansionList( "E -> A" )
+      answer.addExpansionList( [ "A -> digi", Proc.new { print "hi!!!\n" }, "A" ] )
+      answer.addExpansionList( "A -> digi" )
+      answer.addExpansionList( "A -> o A" )
+      answer.addExpansionList( "A -> o" )
+      answer
+    end
+
+    def Language.pageXYZ
+      answer = Language.new
+      answer.addTerminalDescription( "digi", Regexp.compile( /[[:digit:]]/ ) )
+      answer.addTerminalDescription( "o", Regexp.compile( /[abc]/ ) )
+      answer.setNonTerminalNames( "E A" )
+      answer.addExpansionList( "E -> A" )
+      answer.addExpansionList( [ "A -> digi.a digi.d", CCAction.new( 'print "hi #{a} #{d}!!!\n"' ), "A" ] )
+      answer.addExpansionList( "A -> digi" )
+      answer.addExpansionList( "A -> o.num A" )
+      answer.addExpansionList( "A -> o" )
+      answer
+    end
+
+    def Language.addition
+      answer = Language.new
+      answer.addTerminalDescription( "digi", Regexp.compile( /[[:digit:]]/ ) )
+      answer.setTerminalCharacters( "+" )
+      answer.setNonTerminalNames( "E" )
+      answer.addExpansionList( [ "E -> digi.l - digi.r", CCAction.new( 'print "l: #{l} r: #{r} addition: #{l.to_i + r.to_i}\n"' ) ] )
+      answer
+    end
+
+    def Language.regularExpressions
+      answer = Language.new
+      # specials = Array.new
+      specials = "\\ | ^ * + ( ) ?"  # .each_byte { |val| specials.push( val ) }
+      digits = "0 1 2 3 4 5 6 6 7 8 9"
+      answer.setTerminalCharacters( specials )
+      answer.addTerminalDescription( "Digit", Regexp.compile( /[[:digit:]]/ ) )
+      answer.addTerminalDescription( "Char",  Regexp.compile( /[[:alpha:]]/ ) )
+      answer.setNonTerminalNames( "Parenthesized Concatenation Alternative Closure BinaryOperand ClosureOp Literal Op Power UnaryOperand CountingNumber Expression Augmented ContingNum" )
+      answer.addExpansionList( "Augmented -> Expression" )
+      answer.addExpansionList( "Expression -> Concatenation" )
+      answer.addExpansionList( "Expression -> Alternative" )
+      answer.addExpansionList( "Expression -> BinaryOperand" )
+      answer.addExpansionList( "Parenthesized -> ( Expression )" )
+      answer.addExpansionList( "Concatenation -> BinaryOperand BinaryOperand" )
+      answer.addExpansionList( "Concatenation -> BinaryOperand Concatenation" )
+      answer.addExpansionList( "Alternative -> BinaryOperand | BinaryOperand" )
+      answer.addExpansionList( "Alternative -> BinaryOperand | Alternative" )
+      answer.addExpansionList( "Closure -> UnaryOperand ClosureOp" )
+      answer.addExpansionList( "BinaryOperand -> UnaryOperand" )
+      answer.addExpansionList( "BinaryOperand -> Closure" )
+      answer.addExpansionList( "BinaryOperand -> Power" )
+      answer.addExpansionList( "UnaryOperand -> Char" )
+      answer.addExpansionList( "UnaryOperand -> Digit" )
+      answer.addExpansionList( "UnaryOperand -> Literal" )
+      answer.addExpansionList( "UnaryOperand -> Parenthesized" )
+      answer.addExpansionList( "ClosureOp -> *" )
+      answer.addExpansionList( "ClosureOp -> +" )
+      answer.addExpansionList( "ClosureOp -> ?" )
+      answer.addExpansionList( "Literal -> \ Op" )
+      answer.addExpansionList( "Op -> \\" )
+      answer.addExpansionList( "Op -> |" )
+      answer.addExpansionList( "Op -> ^" )
+      answer.addExpansionList( "Op -> ClosureOp" )
+      answer.addExpansionList( "Power -> UnaryOperand ^ CountingNumber" )
+      answer.addExpansionList( "CountingNumber -> Digit" )
+      answer.addExpansionList( "CountingNumber -> Digit CountingNumber" )
+      answer
+    end
+
+    #def Language.expansionDescription
+    #  answer = Language.new
+    #  answer.addTerminalDescription( "word", Regexp.compile( /[[:alpha:]]([[:alpha:]]|[[:digit:]])*/ ) )
+    #  answer.addTerminalDescription( "number", Regexp.compile( /([[:digit]])+/ ) )
+    #  answer.addTerminalTerminalCharacters( "| ->" )
+    #  answer.addExpansionList( "S -> EXPRESSION" )
+    #  answer.addExpansionList( "EXPRESSION -> WORD EXPRESSION" )
+    #  answer.addExpansionList( "EXPRESSION -> endMarker" )
+    #  answer.addExpansionList( "WORD -> word" )
+    #  answer.addExpansionList( "WORD -> word WORD" )
+
+    #  answer.addExpansionList( "word -> words" )
+    #  answer.addExpansionList( "word -> word | word" )
+    #  answer
+    #end
+
+    # definition (declarations)
+    def setNonTerminalName( aSymbol )
+      setNonTerminalNameWithClass( aSymbol, ParseTreeNode.new )
+    end
+    
+    def setNonTerminalNameWithClass( aSymbol, aClass )
+      n = NonTerminal.new
+      n.name = aSymbol
+      n.nodeClass = aClass
+      @nonTerminalDict[ aSymbol ] = n
+    end
+
+    def setNonTerminalNames( namess )
+      if namess.class == String
+	names = namess.split( " " )
+      end
+      names.each { |name| setNonTerminalName( name ) }
+    end
+
+    def setTerminalCharacter( aCharacter )
+      setTerminalNameWithValue( aCharacter, aCharacter )
+    end
+
+    def setTerminalCharacters( aString )
+      aString.split( " " ).each { |b| str = String.new ; str << b ; setTerminalCharacter( str ) if str != " " }
+    end
+
+    def setTerminalNameWithAlternatives( aSymbol, aStringOrCollection )
+      cic = CollectionInclusionComparator.new
+      cic.collection = aStringOrCollection.as_a
+      setTerminalNameWithValue( aSymbol, cic )
+    end
+    
+    def setTerminalNameWithCharacterClass( aSymbol, aStringOrCollection )
+      setTerminalNameWithAlternatives( aSymbol, aStringOrCollection )
+    end
+
+    def setTerminalNameWithCharacterExclusion( aSymbol, aStringOrCollection )
+      setTerminalNameWithExclusion( aSymbol, aStringOrCollection, [ Fixnum ] )
+    end
+
+    def setTerminalNameWithExclusion( aSymbol, aStringOrCollection, classCollection )
+      rcec = RestrictedCollectionExclusionComparator.new
+      rcec.collection = aStringOrCollection.as_a
+      rcec.classes = classCollection
+      setTerminalNameWithValue( aSymbol, rcec )
+    end
+
+    def setTerminalNameWithValue( aSymbol, anObject )
+      t = Terminal.nameAndValue( aSymbol, anObject )
+      t.description = nil # aSymbol
+      @terminalDict[ aSymbol ] = t
+    end
+
+    def addExpansionList( symbols )
+      #print "+addExpansionList+\n"
+      #action( nil, aSequencableCollectionOfSymbols )
+
+      expansionList = symbols
+      
+      grammarString = GrammarString.new
+      alternatives = Array.new
+      actions = Hash.new
+      newExpansionList = Array.new
+      # order matters for shortcuts
+      shortcuts = Array.new
+
+      #print "expansionList: #{expansionList.to_s}\n"
+
+      # find out here how the array is structured..
+      # it can be structed in two ways:
+      # 1. The basic way
+      #  [ "A", "->", "B", "+", "C" ]
+      # 2. With actions
+      #  [ "A -> B", Proc.new { ..... }, "+ C" ]
+      # 3. With actions and/or shortcuts
+      #  [ "A -> B.d", Proc.new { |d| ..... }, "+ C" ]
+      #  [ "A -> B.l + C.r", Proc.new { |l, r| ..... } ]
+
+      ########################## Parse actions and find shortcuts ############################
+      # ridx is the "Real index" of the resulting array .. which can differ from the input
+      # array because of actions
+      ridx = 0
+
+      ### flatten out the expansionList
+      expansionList.each do
+	|symbol|
+	toAdd = symbol if symbol.class == Array or symbol.class == CCAction
+	toAdd = symbol.split( " " ) if symbol.class == String
+	#print "Adding toAdd: #{toAdd.join( " " ).to_s}\n"
+	newExpansionList.addAll( toAdd ) if toAdd.class == Array
+	newExpansionList.push( toAdd ) if toAdd.class == CCAction
+      end
+      expansionList = newExpansionList
+
+      ### get rid of the nonTerminal and -> in the beginning
+      nonTerminal = nonTerminalNamed( expansionList.shift )
+      expansionList.shift if expansionList.first == "->"
+      self.unfinishedExpansionList() if expansionList.empty? == true
+
+      ### parse out the actions, and initialize them
+      newExpansionList = Array.new
+      expansionList.size.times do
+	|idx|
+	symbol = expansionList[ idx ]
+	## take care of the shortcuts. for example, say we have:
+	## [ "A -> B.d", Proc.new { |d| ..... }, "+ C" ] as the input
+	## The first "toAdd" would be "A -> B.d"
+	## This algorithm just splits A -> B.d into another array and prepends
+	## it onto the final array.. and makes sure the shortcuts are in the
+	## right indices
+	if symbol.class != CCAction
+	  toAdd = symbol
+	  if ( splits = symbol.split( "." ) ).size == 2
+	    hmm = nonTerminalDict[ splits.first ]
+	    hmm = terminalDict[ splits.first ] if hmm == nil
+	    error( "Unable to find a terminal/nonTerminal named #{splits.first}" ) if hmm == nil
+	    shortcuts.push( Shortcut.new( ridx, hmm, splits[ 1 ], nil ) )
+	    toAdd = splits.first
+	  end
+	  newExpansionList.push( toAdd )
+	  # since this isn't an action, move the real index along
+	  # we want it to stay the same during an action
+	  ridx = ridx + 1
+	else
+	  action = symbol
+	  index = ridx - 1 if ridx > 0
+	  index = 0 if ridx == 0
+	  action.index = ridx
+	  shortcutsToAdd = shortcuts.select { |shortcut| shortcut.index < ridx }
+	  #print "idx: #{idx} shortcuts to add: #{shortcutsToAdd.join( " " )}\n"
+	  action.setShortcuts( shortcutsToAdd )
+	  action.finalizeAction
+	  actions[ ridx ] = action
+	end
+      end
+      expansionList = newExpansionList
+
+      ########################################################################################
+      ############################# Parse Shortcuts ##########################################
+      ########################################################################################
+
+      symbols = expansionList.collect { |symbol| grammarSymbolNamed( symbol ) }
+
+      alternatives = @productionDict[ nonTerminal ] if @productionDict.has_key?( nonTerminal )
+      @productionDict[ nonTerminal ] = alternatives if !@productionDict.has_key?( nonTerminal )
+
+      grammarString.addAll( symbols )
+      newProduction = Production.new( nonTerminal, actions, shortcuts, grammarString )
+      alternatives.push( newProduction )
+      @startProduction = newProduction if @startProduction == nil
+      #print "-addExpansionList-\n"
+      newProduction
+    end
+
+    def oldExpansionList
+      if toAdd != nil
+	toAdd.size.times do
+	  |idx|
+	  if toAdd[ idx ].class != CCAction
+	    splits = toAdd[ idx ].split( "." )
+	    # if we have something like "D.l"
+	    if splits.size == 2
+	      # shave off the shortcut from toAdd, and store it into shortcuts
+	      shortcuts = Hash.new if shortcuts == nil
+	      shortcutsOrder = Hash.new if shortcutsOrder == nil
+	      # compensate for the "A ->" part in the beginning by shifting left
+	      shortcuts[ ridx ] = splits[ -1 ]
+	      shortcutsOrder[ shortcuts[ ridx ] ] = shortcutIdx
+	      # for a string like "A.d", just add "A" onto toAdd
+	      toAdd[ idx ] = splits[ 0 ]
+	      shortcutIdx = shortcutIdx + 1
+	    end
+	    ridx = ridx + 1
+	  end
+	end
+	#print "ridx is: #{ridx}\n"
+	#print "toAdd is: #{toAdd.join( "  and  " )}\n"
+	#print "shortcuts: #{shortcuts.keys.join( "  and  " )} #{shortcuts.values.join( "  and  " )}\n and shortcutsOrder: #{shortcutsOrder.keys.join( "  and  " )}\n" if shortcuts != nil
+      end
+      # newExpansionList.addAll( toAdd ) if symbol.class == Array or symbol.class == String
+      newExpansionList.addAll( toAdd ) if toAdd.class == Array or toAdd.class == String
+      
+      ## take care of the actions
+      if symbol.class == CCAction
+	symbol.index = ridx
+	symbol.setShortcuts( shortcuts.invert, shortcutsOrder )
+	symbol.finalizeAction
+	actions[ ridx - lshift ] = symbol
+	#print "set action to index: #{ridx - lshift} with symbol: #{symbol} ridx is: #{ridx}\n"
+	ridx = ridx - 1
+	#exit
+      end
+      ridx = ridx + 1
+      #print "Actions size: #{actions.size}\n"
+      actions.each_pair { |key,value| "key: #{key} value: #{value}\n" } if actions.size > 0    
+      $defout.flush
+      #exit
+      expansionList = newExpansionList if newExpansionList.size > 0
+      expansionList = expansionList.split( " " ) if expansionList.class == String
+    end
+
+    # errors
+    def redundantAugmentation; self.error( "This language has already been augmented (or the augmentedStart symbol used previously" ); end  
+    def unfinishedExpansionList; self.error( "Unfinished expansionList" ); end
+    def unregisteredGrammarSymbolNamed( aSymbol ); self.error( "Reference to unregistered GrammarSymbol named: #{aSymbol}" ); end
+    def unregisteredNonTerminalNamed( aSymbol ); self.error( "Error: Reference to unregistered NonTerminal named: #{aSymbol}" ); end
+    def unregisteredTerminalNamed( aSymbol ); self.error( "Reference to unregistered Terminal named #{aSymbol}" ); end
+    
+    # lookups
+    def grammarSymbolNamed( aSymbol )
+      return testAbsenseForGrammarSymbolNamed( aSymbol ) { unregisteredGrammarSymbolNamed( aSymbol ) }
+    end
+    
+    def testAbsenseForGrammarSymbolNamed( aSymbol, &noArgBlock )
+      #print "Testing absense for grammar symbol named: #{aSymbol}\n"
+      return testAbsenseForNonTerminalNamed( aSymbol ) { return testAbsenseForTerminalNamed( aSymbol ) { noArgBlock.call } }
+    end
+    
+    def nonTerminalNamed( aSymbol )
+      return testAbsenseForNonTerminalNamed( aSymbol ) { unregisteredNonTerminalNamed( aSymbol ) }
+    end
+    
+    def testAbsenseForNonTerminalNamed( aSymbol, &noArgBlock )
+      #@nonTerminalDict[ aSymbol ] = noArgBlock if @nonTerminalDict[ aSymbol ] == nil
+      #@nonTerminalDict.each_pair { |key, value| print "Key: |#{key}| and value: |#{value}|\n" }
+      noArgBlock.call if !@nonTerminalDict.has_key?( aSymbol )
+      @nonTerminalDict[ aSymbol ]
+    end
+    
+    def productionsForNonTerminal( aNonTerminal )
+      @productionDict[ aNonTerminal ]
+    end
+    
+    def terminalNamed( aSymbol )
+      testAbsenseForTerminalNamed( aSymbol ) { unregisteredTerminalNamed( aSymbol ) }
+    end
+    
+    def testAbsenseForTerminalNamed( aSymbol, &noArgBlock )
+      #print "testing absense for terminal: #{aSymbol}\n"
+      #print "printing dict now, symbol is: #{aSymbol}: \n"
+      #print "its value: #{@terminalDict[aSymbol.to_sym]}\n"
+      #@terminalDict.each_pair { |key, value| print "Key: |#{key}| and value: |#{value}|\n" }
+      noArgBlock.call if @terminalDict.has_key?( aSymbol ) == false
+      #print "pretty printing it: "
+      #pp @terminalDict[ aSymbol ]
+      return @terminalDict[ aSymbol ]
+    end
+    
+    # operations
+    def augment
+      # Converts the language into an augmented language
+      # see page 222, 'compilers: principles, techniques, and tools' by aho, sethi, and ullman
+      if !isDefined?
+	notDefined
+      end
+      if @terminalDict.has_key?( augmentedStartName )
+	redundantAugmentation
+      end
+      setNonTerminalName( augmentedStartName )
+      @startProduction = addExpansionList( [ augmentedStartName, @startProduction.nonTerminal.name ] ) 
+    end
+
+    def augmentedStartName
+      "AugmentedStart"
+    end
+
+    def errorCycle_first
+      # this is an extremely private method...
+      processor = Processor.new
+      processor.frameStackSize = 200
+      processor.methodNamed( "unprotectedFirst" )
+      processor.processedBy = Proc.new { 
+	|frames| 
+	continue = true
+	answer = Array.new
+	frames.each {
+	  |frame|
+	  if continue == true
+	    newArg = frame.arguments.first
+	    answer.unshift( newArg )
+	    continue = ( answer.size < 2 or answer.last =~ newArg )
+	  end
+	}
+	answer
+      }
+    end
+
+    def first( aGrammarStringOrGrammarSymbol )
+      #print "in first, type is: #{aGrammarStringOrGrammarSymbol.class.to_s}\n"
+      answer = setUnprotectedFirst( aGrammarStringOrGrammarSymbol )
+      answer
+    end
+
+    def follow( nonTerminal )
+      @followsDict[ nonTerminal ] = nonTerminal.followsWithinLanguage( self ) if !@followsDict.has_key?( nonTerminal )
+      @followsDict[ nonTerminal ]
+    end
+    
+    def unprotectedFirst( aGrammarStringOrGrammarSymbol )
+      #print "unprotectedfirst.... printing it: \n"
+      #@firstDict.each_pair { |one, two| print "first: #{one.to_s} has type: #{one.class.to_s} last: #{two.to_a.join( "  and  " )}\n" }
+      @firstDict[ aGrammarStringOrGrammarSymbol ] = aGrammarStringOrGrammarSymbol.firstWithinLanguage( self ) if @firstDict[ aGrammarStringOrGrammarSymbol ] == nil
+      @firstDict[ aGrammarStringOrGrammarSymbol ]
+    end
+
+    # query
+    def grammarSymbolNames; nonTerminalNames.join( " " ) + " " + terminalNames.join( " " ); end
+
+    def grammarSymbols
+      nts = nonTerminals
+      #print "nts is: #{nts.join( " " )} include A: " + nts.join( " " ).include?( "A" ).to_s + "\n"
+      terms = terminals #.join( " " ).to_a
+      array = nts.concat( terms )
+      array.join( " " )
+    end
+
+    def isDefined?; @startProduction != nil; end  
+    def nonTerminalNames; @nonTerminalDict.keys.to_itemSet; end
+    def nonTerminals; @nonTerminalDict.values.to_itemSet; end
+
+    def productions
+      answer = Array.new
+      @productionDict.values each { |value| answer.addAll( value ) }
+      answer
+    end
+
+    def productionsFromNonTerminal( aGrammarSymbol )
+      val = @productionDict[ aGrammarSymbol ]
+      return val if val != nil
+      error( "Missing production for " + aGrammarSymbol.name )
+    end
+
+    def productionsWithExpansionContainingNonTerminal( aGrammarSymbol )
+      answer = Array.new
+      #print "\nproductiondict's size: #{@productionDict.size}\n"
+      #print "its values: #{@productionDict.values.join( "  and  " )}\n"
+      @productionDict.values.each do
+	|collection|
+	#print "looking at: #{collection.join( " and  " )}\n"
+	collection.each do
+	  |production|
+	  if production.expansionContainsNonTerminal?( aGrammarSymbol )
+	    answer.push( production )
+	  end
+	end
+      end
+      #print "answer: "
+      #answer.each { |ans| print "#{ans.to_s}  and  " }
+      #print "\n"
+      answer
+    end
+
+    def startingNonTerminal; @startProduction.nonTerminal; end
+    def terminalNames; @terminalDict.keys.to_itemSet; end
+    def terminals; @terminalDict.values.to_itemSet; end
+
+    def setUnprotectedFirst( aGrammarStringOrGrammarSymbol )
+      #print "firstDict is: #{@firstDict.to_a.join( "  and  " )}\n"
+      @firstDict.testAbsenseFor( aGrammarStringOrGrammarSymbol.to_s ) { @firstDict[ aGrammarStringOrGrammarSymbol ] = aGrammarStringOrGrammarSymbol.firstWithinLanguage( self ) }
+      @firstDict[ aGrammarStringOrGrammarSymbol ]
+    end
+
+    def finalizeFirstAndFollow
+      #print "\n@@@@@@2nonTerminalDict has #{nonTerminalDict.values.size.to_s} values\n"
+      @terminalDict.values.each do
+	|terminal| 
+	first( terminal )
+	#print "firstDict: #{@firstDict.to_a.join( "  and  " )}\n"
+      end
+      @nonTerminalDict.values.each do
+	|nonTerminal| 
+	#print "doing nonTerminal #{nonTerminal}\n"
+	#print "**firstDict: "
+	first( nonTerminal )
+	#@firstDict.each_pair { |key,value| print "^key: #{key} and value: #{value}   " }
+	#print "\n"
+	follow( nonTerminal )
+      end
+    end
+
+    def addTerminalDescription( name, pattern )
+      terminal = setTerminalNameWithValue( name, "" )
+      terminal.description = pattern
+      #@terminalDescriptions[ pattern ] = terminal
+    end
+
+  end
+end