taskjuggler 0.0.7 → 0.0.8

data/doc/TaskJuggler/TextParser/Pattern.html

@@ -75,10 +75,14 @@
 					
 					<li><a href="#method-i-addArgDoc">#addArgDoc</a></li>
 					
+					<li><a href="#method-i-addTransitionsToState">#addTransitionsToState</a></li>
+					
 					<li><a href="#method-i-each">#each</a></li>
 					
 					<li><a href="#method-i-empty%3F">#empty?</a></li>
 					
+					<li><a href="#method-i-generateStates">#generateStates</a></li>
+					
 					<li><a href="#method-i-length">#length</a></li>
 					
 					<li><a href="#method-i-optional%3F">#optional?</a></li>
@@ -181,12 +185,18 @@
 				
 					<li><a href="../../TaskJuggler/ChargeSetListAttribute.html">TaskJuggler::ChargeSetListAttribute</a></li>
 				
+					<li><a href="../../TaskJuggler/CollisionDetector.html">TaskJuggler::CollisionDetector</a></li>
+				
 					<li><a href="../../TaskJuggler/ColumnListAttribute.html">TaskJuggler::ColumnListAttribute</a></li>
 				
 					<li><a href="../../TaskJuggler/ColumnTable.html">TaskJuggler::ColumnTable</a></li>
 				
 					<li><a href="../../TaskJuggler/Daemon.html">TaskJuggler::Daemon</a></li>
 				
+					<li><a href="../../TaskJuggler/DataCache.html">TaskJuggler::DataCache</a></li>
+				
+					<li><a href="../../TaskJuggler/DataCacheEntry.html">TaskJuggler::DataCacheEntry</a></li>
+				
 					<li><a href="../../TaskJuggler/DateAttribute.html">TaskJuggler::DateAttribute</a></li>
 				
 					<li><a href="../../TaskJuggler/DefinitionListAttribute.html">TaskJuggler::DefinitionListAttribute</a></li>
@@ -293,7 +303,7 @@
 				
 					<li><a href="../../TaskJuggler/NodeListAttribute.html">TaskJuggler::NodeListAttribute</a></li>
 				
-					<li><a href="../../TaskJuggler/OnShiftCache.html">TaskJuggler::OnShiftCache</a></li>
+					<li><a href="../../TaskJuggler/PlaceHolderCell.html">TaskJuggler::PlaceHolderCell</a></li>
 				
 					<li><a href="../../TaskJuggler/ProcessIntercom.html">TaskJuggler::ProcessIntercom</a></li>
 				
@@ -473,6 +483,10 @@
 				
 					<li><a href="../../TaskJuggler/TextParser/StackElement.html">TaskJuggler::TextParser::StackElement</a></li>
 				
+					<li><a href="../../TaskJuggler/TextParser/State.html">TaskJuggler::TextParser::State</a></li>
+				
+					<li><a href="../../TaskJuggler/TextParser/StateTransition.html">TaskJuggler::TextParser::StateTransition</a></li>
+				
 					<li><a href="../../TaskJuggler/TextParser/TextParserResultArray.html">TaskJuggler::TextParser::TextParserResultArray</a></li>
 				
 					<li><a href="../../TaskJuggler/TextParser/TokenDoc.html">TaskJuggler::TextParser::TokenDoc</a></li>
@@ -586,21 +600,21 @@ are Strings where the first character determines the type of the token.
 There are 4 known types.
 </p>
 <p>
-Terminal token: The terminal token is prefixed by an underscore. Terminal
-tokens are terminal symbols of the syntax tree. They just represent
-themselves.
+Terminal token: In the syntax declaration the terminal token is prefixed by
+an underscore. Terminal tokens are terminal symbols of the syntax tree.
+They just represent themselves.
 </p>
 <p>
 Variable token: The variable token describes values of a certain class such
-as strings or numbers. The token is prefixed by a dollar sign and the text
-of the token specifies the variable type. See <a
+as strings or numbers. In the syntax declaration the token is prefixed by a
+dollar sign and the text of the token specifies the variable type. See <a
 href="../ProjectFileParser.html">ProjectFileParser</a> for a complete list
 of variable types.
 </p>
 <p>
 Reference token: The reference token specifies a reference to another
-parser rule. The token is prefixed by a bang and the text matches the name
-of the rule. See TextParserRule for details.
+parser rule. In the syntax declaration the token is prefixed by a bang and
+the text matches the name of the rule. See TextParserRule for details.
 </p>
 <p>
 End token: The . token marks the expected end of the input stream.
@@ -755,37 +769,55 @@ documentary information about the pattern.
 					<div class="method-source-code"
 						id="new-source">
 <pre>
-    <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 44</span>
-44:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">tokens</span>, <span class="ruby-identifier">function</span> = <span class="ruby-keyword kw">nil</span>)
-45:       <span class="ruby-comment cmt"># A unique name for the pattern that is used in the documentation.</span>
-46:       <span class="ruby-ivar">@keyword</span> = <span class="ruby-keyword kw">nil</span>
-47:       <span class="ruby-comment cmt"># Initialize pattern doc as empty.</span>
-48:       <span class="ruby-ivar">@doc</span> = <span class="ruby-keyword kw">nil</span>
-49:       <span class="ruby-comment cmt"># A list of TokenDoc elements that describe the meaning of variable</span>
-50:       <span class="ruby-comment cmt"># tokens. The order of the tokens and entries in the Array must correlate.</span>
-51:       <span class="ruby-ivar">@args</span> = []
-52:       <span class="ruby-comment cmt"># A list of references to other patterns that are related to this pattern.</span>
-53:       <span class="ruby-ivar">@seeAlso</span> = []
-54:       <span class="ruby-comment cmt"># A reference to a file under test/TestSuite/Syntax/Correct and a tag</span>
-55:       <span class="ruby-comment cmt"># within that file. This identifies example TJP code to be included with</span>
-56:       <span class="ruby-comment cmt"># the reference manual.</span>
-57:       <span class="ruby-ivar">@exampleFile</span> = <span class="ruby-keyword kw">nil</span>
-58:       <span class="ruby-ivar">@exampleTag</span> = <span class="ruby-keyword kw">nil</span>
-59: 
-60:       <span class="ruby-identifier">tokens</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">token</span><span class="ruby-operator">|</span>
-61:         <span class="ruby-keyword kw">unless</span> <span class="ruby-value str">'!$_.'</span>.<span class="ruby-identifier">include?</span>(<span class="ruby-identifier">token</span>[<span class="ruby-value">0</span>])
-62:           <span class="ruby-identifier">raise</span> <span class="ruby-value str">&quot;Fatal Error: All pattern tokens must start with a type &quot;</span> <span class="ruby-operator">+</span>
-63:                 <span class="ruby-node">&quot;identifier [!$_.]: #{tokens.join(', ')}&quot;</span>
-64:         <span class="ruby-keyword kw">end</span>
-65:         <span class="ruby-comment cmt"># Initialize pattern argument descriptions as empty.</span>
-66:         <span class="ruby-ivar">@args</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-keyword kw">nil</span>
-67:       <span class="ruby-keyword kw">end</span>
-68:       <span class="ruby-ivar">@tokens</span> = <span class="ruby-identifier">tokens</span>
-69:       <span class="ruby-ivar">@function</span> = <span class="ruby-identifier">function</span>
-70:       <span class="ruby-comment cmt"># In some cases we don't want to show all tokens in the syntax</span>
-71:       <span class="ruby-comment cmt"># documentation. This value specifies the index of the last shown token.</span>
-72:       <span class="ruby-ivar">@lastSyntaxToken</span> = <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">-</span> <span class="ruby-value">1</span>
-73:     <span class="ruby-keyword kw">end</span></pre>
+    <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 45</span>
+45:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">tokens</span>, <span class="ruby-identifier">function</span> = <span class="ruby-keyword kw">nil</span>)
+46:       <span class="ruby-comment cmt"># A unique name for the pattern that is used in the documentation.</span>
+47:       <span class="ruby-ivar">@keyword</span> = <span class="ruby-keyword kw">nil</span>
+48:       <span class="ruby-comment cmt"># Initialize pattern doc as empty.</span>
+49:       <span class="ruby-ivar">@doc</span> = <span class="ruby-keyword kw">nil</span>
+50:       <span class="ruby-comment cmt"># A list of TokenDoc elements that describe the meaning of variable</span>
+51:       <span class="ruby-comment cmt"># tokens. The order of the tokens and entries in the Array must correlate.</span>
+52:       <span class="ruby-ivar">@args</span> = []
+53:       <span class="ruby-comment cmt"># A list of references to other patterns that are related to this pattern.</span>
+54:       <span class="ruby-ivar">@seeAlso</span> = []
+55:       <span class="ruby-comment cmt"># A reference to a file under test/TestSuite/Syntax/Correct and a tag</span>
+56:       <span class="ruby-comment cmt"># within that file. This identifies example TJP code to be included with</span>
+57:       <span class="ruby-comment cmt"># the reference manual.</span>
+58:       <span class="ruby-ivar">@exampleFile</span> = <span class="ruby-keyword kw">nil</span>
+59:       <span class="ruby-ivar">@exampleTag</span> = <span class="ruby-keyword kw">nil</span>
+60: 
+61:       <span class="ruby-ivar">@tokens</span> = []
+62:       <span class="ruby-identifier">tokens</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">token</span><span class="ruby-operator">|</span>
+63:         <span class="ruby-keyword kw">unless</span> <span class="ruby-value str">'!$_.'</span>.<span class="ruby-identifier">include?</span>(<span class="ruby-identifier">token</span>[<span class="ruby-value">0</span>])
+64:           <span class="ruby-identifier">raise</span> <span class="ruby-value str">&quot;Fatal Error: All pattern tokens must start with a type &quot;</span> <span class="ruby-operator">+</span>
+65:                 <span class="ruby-node">&quot;identifier [!$_.]: #{tokens.join(', ')}&quot;</span>
+66:         <span class="ruby-keyword kw">end</span>
+67:         <span class="ruby-comment cmt"># For the syntax specification using a prefix character is more</span>
+68:         <span class="ruby-comment cmt"># convenient. But for further processing, we need to split the string</span>
+69:         <span class="ruby-comment cmt"># into two symbols. The prefix determines the token type, the rest is</span>
+70:         <span class="ruby-comment cmt"># the token name. There are 4 types of tokens:</span>
+71:         <span class="ruby-comment cmt"># :reference : a reference to another rule</span>
+72:         <span class="ruby-comment cmt"># :variable : a terminal symbol</span>
+73:         <span class="ruby-comment cmt"># :literal : a user defined string</span>
+74:         <span class="ruby-comment cmt"># :eof : marks the end of an input stream</span>
+75:         <span class="ruby-identifier">type</span> = [ <span class="ruby-value">:reference</span>, <span class="ruby-value">:variable</span>, <span class="ruby-value">:literal</span>, <span class="ruby-value">:eof</span> ][<span class="ruby-value str">'!$_.'</span>.<span class="ruby-identifier">index</span>(<span class="ruby-identifier">token</span>[<span class="ruby-value">0</span>])]
+76:         <span class="ruby-comment cmt"># For literals we use a String to store the token content. For others,</span>
+77:         <span class="ruby-comment cmt"># a symbol is better suited.</span>
+78:         <span class="ruby-identifier">name</span> = <span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:literal</span> <span class="ruby-operator">?</span>
+79:                <span class="ruby-identifier">token</span>[<span class="ruby-value">1</span><span class="ruby-operator">..</span><span class="ruby-value">1</span>] <span class="ruby-operator">:</span> (<span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:eof</span> <span class="ruby-operator">?</span> <span class="ruby-value str">'&lt;END&gt;'</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">token</span>[<span class="ruby-value">1</span><span class="ruby-operator">..</span><span class="ruby-value">1</span>].<span class="ruby-identifier">intern</span>)
+80:         <span class="ruby-comment cmt"># We favor an Array to store the 2 elements over a Hash for</span>
+81:         <span class="ruby-comment cmt"># performance reasons.</span>
+82:         <span class="ruby-ivar">@tokens</span> <span class="ruby-operator">&lt;&lt;</span> [ <span class="ruby-identifier">type</span>, <span class="ruby-identifier">name</span> ]
+83:         <span class="ruby-comment cmt"># Initialize pattern argument descriptions as empty.</span>
+84:         <span class="ruby-ivar">@args</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-keyword kw">nil</span>
+85:       <span class="ruby-keyword kw">end</span>
+86:       <span class="ruby-ivar">@function</span> = <span class="ruby-identifier">function</span>
+87:       <span class="ruby-comment cmt"># In some cases we don't want to show all tokens in the syntax</span>
+88:       <span class="ruby-comment cmt"># documentation. This value specifies the index of the last shown token.</span>
+89:       <span class="ruby-ivar">@lastSyntaxToken</span> = <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">-</span> <span class="ruby-value">1</span>
+90: 
+91:       <span class="ruby-ivar">@transitions</span> = []
+92:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -824,10 +856,123 @@ Conveniance function to access individual tokens by index.
 					<div class="method-source-code"
 						id="--source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 103</span>
-103:     <span class="ruby-keyword kw">def</span> <span class="ruby-operator">[]</span>(<span class="ruby-identifier">i</span>)
-104:       <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">i</span>]
-105:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 210</span>
+210:     <span class="ruby-keyword kw">def</span> <span class="ruby-operator">[]</span>(<span class="ruby-identifier">i</span>)
+211:       <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">i</span>]
+212:     <span class="ruby-keyword kw">end</span></pre>
+					</div>
+					
+				</div>
+
+				
+
+				
+			</div>
+
+		
+			<div id="add-ransitions-o-tate-method" class="method-detail ">
+				<a name="method-i-addTransitionsToState"></a>
+
+				<div class="method-heading">
+				
+					<span class="method-name">addTransitionsToState</span><span
+						class="method-args">(states, rules, stateStack, sourceState, destRule, destIndex, loopBack)</span>
+					<span class="method-click-advice">click to toggle source</span>
+				
+				</div>
+
+				<div class="method-description">
+					
+					<p>
+Add the transitions to the <a href="State.html">State</a> objects of this
+pattern. <em>states</em> is a Hash with all <a href="State.html">State</a>
+objects. <em>rules</em> is a Hash with the <a href="Rule.html">Rule</a>
+objects of the syntax. <em>stateStack</em> is an Array of <a
+href="State.html">State</a> objects that have been traversed before
+reaching this pattern. <em>sourceState</em> is the <a
+href="State.html">State</a> that the transition originates from.
+<em>destRule</em>, this pattern and <em>destIndex</em> describe the <a
+href="State.html">State</a> the transition is leading to. <em>loopBack</em>
+is boolean flag, set to true when the transition describes a loop back to
+the start of the <a href="Rule.html">Rule</a>.
+</p>
+					
+
+					
+					<div class="method-source-code"
+						id="add-ransitions-o-tate-source">
+<pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 111</span>
+111:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">addTransitionsToState</span>(<span class="ruby-identifier">states</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-identifier">stateStack</span>, <span class="ruby-identifier">sourceState</span>,
+112:                               <span class="ruby-identifier">destRule</span>, <span class="ruby-identifier">destIndex</span>, <span class="ruby-identifier">loopBack</span>)
+113:       <span class="ruby-comment cmt"># If we hit a token in the pattern that is optional, we need to consider</span>
+114:       <span class="ruby-comment cmt"># the next token of the pattern as well.</span>
+115:       <span class="ruby-identifier">loop</span> <span class="ruby-keyword kw">do</span>
+116:         <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">destIndex</span> <span class="ruby-operator">&gt;=</span> <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">length</span>
+117:           <span class="ruby-comment cmt"># Have we reached the end of the pattern? Such state always trigger</span>
+118:           <span class="ruby-comment cmt"># a reduce operation.</span>
+119:           <span class="ruby-identifier">sourceState</span>.<span class="ruby-identifier">noReduce</span> = <span class="ruby-keyword kw">false</span>
+120:           <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">sourceState</span>.<span class="ruby-identifier">rule</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">destRule</span>
+121:             <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">destRule</span>.<span class="ruby-identifier">repeatable</span>
+122:               <span class="ruby-comment cmt"># The transition leads us back to the start of the Rule. This</span>
+123:               <span class="ruby-comment cmt"># will generate transitions to the first token of all patterns</span>
+124:               <span class="ruby-comment cmt"># of this Rule.</span>
+125:               <span class="ruby-identifier">destRule</span>.<span class="ruby-identifier">addTransitionsToState</span>(<span class="ruby-identifier">states</span>, <span class="ruby-identifier">rules</span>, [], <span class="ruby-identifier">sourceState</span>,
+126:                                              <span class="ruby-keyword kw">true</span>)
+127:             <span class="ruby-keyword kw">end</span>
+128:           <span class="ruby-keyword kw">end</span>
+129:           <span class="ruby-comment cmt"># We've reached the end of the pattern. No more transitions to</span>
+130:           <span class="ruby-comment cmt"># consider.</span>
+131:           <span class="ruby-keyword kw">return</span>
+132:         <span class="ruby-keyword kw">end</span>
+133: 
+134:         <span class="ruby-comment cmt"># The token descriptor tells us where the transition(s) need to go to.</span>
+135:         <span class="ruby-identifier">tokenType</span>, <span class="ruby-identifier">tokenName</span> = <span class="ruby-identifier">token</span> = <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">destIndex</span>]
+136: 
+137:         <span class="ruby-keyword kw">case</span> <span class="ruby-identifier">tokenType</span>
+138:         <span class="ruby-keyword kw">when</span> <span class="ruby-value">:reference</span>
+139:           <span class="ruby-comment cmt"># The descriptor references another rule.</span>
+140:           <span class="ruby-keyword kw">unless</span> (<span class="ruby-identifier">refRule</span> = <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">tokenName</span>])
+141:             <span class="ruby-identifier">raise</span> <span class="ruby-node">&quot;Unknown rule #{tokenName} referenced in rule #{refRule.name}&quot;</span>
+142:           <span class="ruby-keyword kw">end</span>
+143:           <span class="ruby-comment cmt"># If we reference another rule from a pattern, we need to come back</span>
+144:           <span class="ruby-comment cmt"># to the pattern once we are done with the referenced rule. To be</span>
+145:           <span class="ruby-comment cmt"># able to come back, we collect a list of all the States that we</span>
+146:           <span class="ruby-comment cmt"># have passed during a reference resolution. This list forms a stack</span>
+147:           <span class="ruby-comment cmt"># that is popped during recude operations of the parser FSM.</span>
+148:           <span class="ruby-identifier">skippedState</span> = <span class="ruby-identifier">states</span>[[ <span class="ruby-identifier">destRule</span>, <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">destIndex</span> ]]
+149:           <span class="ruby-comment cmt"># Rules may reference themselves directly or indirectly. To avoid</span>
+150:           <span class="ruby-comment cmt"># endless recursions of this algorithm, we stop once we have</span>
+151:           <span class="ruby-comment cmt"># detected a recursion. We have already all necessary transitions</span>
+152:           <span class="ruby-comment cmt"># collected. The recursion will be unrolled in the parser FSM.</span>
+153:           <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">stateStack</span>.<span class="ruby-identifier">include?</span>(<span class="ruby-identifier">skippedState</span>)
+154:             <span class="ruby-comment cmt"># Push the skipped state on the stateStack before recursing.</span>
+155:             <span class="ruby-identifier">stateStack</span>.<span class="ruby-identifier">push</span>(<span class="ruby-identifier">skippedState</span>)
+156:             <span class="ruby-identifier">refRule</span>.<span class="ruby-identifier">addTransitionsToState</span>(<span class="ruby-identifier">states</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-identifier">stateStack</span>,
+157:                                           <span class="ruby-identifier">sourceState</span>, <span class="ruby-identifier">loopBack</span>)
+158:             <span class="ruby-comment cmt"># Once we're done, remove the State from the stateStack again.</span>
+159:             <span class="ruby-identifier">stateStack</span>.<span class="ruby-identifier">pop</span>
+160:           <span class="ruby-keyword kw">end</span>
+161: 
+162:           <span class="ruby-comment cmt"># If the referenced rule is not optional, we have no further</span>
+163:           <span class="ruby-comment cmt"># transitions for this pattern at this destIndex.</span>
+164:           <span class="ruby-keyword kw">break</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">refRule</span>.<span class="ruby-identifier">optional?</span>(<span class="ruby-identifier">rules</span>)
+165:         <span class="ruby-keyword kw">else</span>
+166:           <span class="ruby-keyword kw">unless</span> (<span class="ruby-identifier">destState</span> = <span class="ruby-identifier">states</span>[[ <span class="ruby-identifier">destRule</span>, <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">destIndex</span> ]])
+167:             <span class="ruby-identifier">raise</span> <span class="ruby-value str">&quot;Destination state not found&quot;</span>
+168:           <span class="ruby-keyword kw">end</span>
+169:           <span class="ruby-comment cmt"># We've found a transition to a terminal token. Add the transition</span>
+170:           <span class="ruby-comment cmt"># to the source State.</span>
+171:           <span class="ruby-identifier">sourceState</span>.<span class="ruby-identifier">addTransition</span>(<span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">destIndex</span>], <span class="ruby-identifier">destState</span>, <span class="ruby-identifier">stateStack</span>,
+172:                                     <span class="ruby-identifier">loopBack</span>)
+173:           <span class="ruby-comment cmt"># Fixed tokens are never optional. There are no more transitions for</span>
+174:           <span class="ruby-comment cmt"># this pattern at this index.</span>
+175:           <span class="ruby-keyword kw">break</span>
+176:         <span class="ruby-keyword kw">end</span>
+177: 
+178:         <span class="ruby-identifier">destIndex</span> <span class="ruby-operator">+=</span> <span class="ruby-value">1</span>
+179:       <span class="ruby-keyword kw">end</span>
+180:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -860,10 +1005,10 @@ Iterator for tokens.
 					<div class="method-source-code"
 						id="each-source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 108</span>
-108:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">each</span>
-109:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">tok</span><span class="ruby-operator">|</span> <span class="ruby-keyword kw">yield</span> <span class="ruby-identifier">tok</span> }
-110:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 215</span>
+215:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">each</span>
+216:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">type</span>, <span class="ruby-identifier">name</span><span class="ruby-operator">|</span> <span class="ruby-keyword kw">yield</span>(<span class="ruby-identifier">type</span>, <span class="ruby-identifier">name</span>) }
+217:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -896,10 +1041,50 @@ Returns true of the pattern is empty.
 					<div class="method-source-code"
 						id="empty--source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 113</span>
-113:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">empty?</span>
-114:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">empty?</span>
-115:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 220</span>
+220:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">empty?</span>
+221:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">empty?</span>
+222:     <span class="ruby-keyword kw">end</span></pre>
+					</div>
+					
+				</div>
+
+				
+
+				
+			</div>
+
+		
+			<div id="generate-tates-method" class="method-detail ">
+				<a name="method-i-generateStates"></a>
+
+				<div class="method-heading">
+				
+					<span class="method-name">generateStates</span><span
+						class="method-args">(rule)</span>
+					<span class="method-click-advice">click to toggle source</span>
+				
+				</div>
+
+				<div class="method-description">
+					
+					<p>
+Generate the state machine states for the pattern. <em>rule</em> is the <a
+href="Rule.html">Rule</a> that the pattern belongs to. A list of generated
+<a href="State.html">State</a> objects will be returned.
+</p>
+					
+
+					
+					<div class="method-source-code"
+						id="generate-tates-source">
+<pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 97</span>
+ 97:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">generateStates</span>(<span class="ruby-identifier">rule</span>)
+ 98:       <span class="ruby-identifier">states</span> = []
+ 99:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">length</span>.<span class="ruby-identifier">times</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">i</span><span class="ruby-operator">|</span> <span class="ruby-identifier">states</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-constant">State</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">rule</span>, <span class="ruby-keyword kw">self</span>, <span class="ruby-identifier">i</span>) }
+100:       <span class="ruby-identifier">states</span>
+101:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -932,10 +1117,10 @@ Returns the number of tokens in the pattern.
 					<div class="method-source-code"
 						id="length-source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 118</span>
-118:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">length</span>
-119:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">length</span>
-120:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 225</span>
+225:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">length</span>
+226:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">length</span>
+227:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -969,20 +1154,19 @@ references a rule, this rule is followed for the check.
 					<div class="method-source-code"
 						id="optional--source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 124</span>
-124:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">optional?</span>(<span class="ruby-identifier">rules</span>)
-125:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">token</span><span class="ruby-operator">|</span>
-126:         <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">token</span>[<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">__</span> <span class="ruby-operator">||</span> <span class="ruby-identifier">token</span>[<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">$$</span>
-127:           <span class="ruby-keyword kw">return</span> <span class="ruby-keyword kw">false</span>
-128:         <span class="ruby-keyword kw">elsif</span> <span class="ruby-identifier">token</span>[<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">!!</span>
-129:           <span class="ruby-identifier">token</span> = <span class="ruby-identifier">token</span>[<span class="ruby-value">1</span><span class="ruby-operator">..</span><span class="ruby-value">1</span>]
-130:           <span class="ruby-keyword kw">if</span> <span class="ruby-operator">!</span><span class="ruby-identifier">rules</span>[<span class="ruby-identifier">token</span>].<span class="ruby-identifier">optional?</span>(<span class="ruby-identifier">rules</span>)
-131:             <span class="ruby-keyword kw">return</span> <span class="ruby-keyword kw">false</span>
-132:           <span class="ruby-keyword kw">end</span>
-133:         <span class="ruby-keyword kw">end</span>
-134:       <span class="ruby-keyword kw">end</span>
-135:       <span class="ruby-keyword kw">true</span>
-136:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 231</span>
+231:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">optional?</span>(<span class="ruby-identifier">rules</span>)
+232:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">type</span>, <span class="ruby-identifier">name</span><span class="ruby-operator">|</span>
+233:         <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:literal</span> <span class="ruby-operator">||</span> <span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:variable</span>
+234:           <span class="ruby-keyword kw">return</span> <span class="ruby-keyword kw">false</span>
+235:         <span class="ruby-keyword kw">elsif</span> <span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:reference</span>
+236:           <span class="ruby-keyword kw">if</span> <span class="ruby-operator">!</span><span class="ruby-identifier">rules</span>[<span class="ruby-identifier">name</span>].<span class="ruby-identifier">optional?</span>(<span class="ruby-identifier">rules</span>)
+237:             <span class="ruby-keyword kw">return</span> <span class="ruby-keyword kw">false</span>
+238:           <span class="ruby-keyword kw">end</span>
+239:         <span class="ruby-keyword kw">end</span>
+240:       <span class="ruby-keyword kw">end</span>
+241:       <span class="ruby-keyword kw">true</span>
+242:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1015,10 +1199,10 @@ Set the documentation text and for the idx-th variable.
 					<div class="method-source-code"
 						id="set-rg-source">
 <pre>
-    <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 82</span>
-82:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setArg</span>(<span class="ruby-identifier">idx</span>, <span class="ruby-identifier">doc</span>)
-83:       <span class="ruby-ivar">@args</span>[<span class="ruby-identifier">idx</span>] = <span class="ruby-identifier">doc</span>
-84:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 189</span>
+189:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setArg</span>(<span class="ruby-identifier">idx</span>, <span class="ruby-identifier">doc</span>)
+190:       <span class="ruby-ivar">@args</span>[<span class="ruby-identifier">idx</span>] = <span class="ruby-identifier">doc</span>
+191:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1051,11 +1235,11 @@ Set the keyword and documentation text for the pattern.
 					<div class="method-source-code"
 						id="set-oc-source">
 <pre>
-    <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 76</span>
-76:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setDoc</span>(<span class="ruby-identifier">keyword</span>, <span class="ruby-identifier">doc</span>)
-77:       <span class="ruby-ivar">@keyword</span> = <span class="ruby-identifier">keyword</span>
-78:       <span class="ruby-ivar">@doc</span> = <span class="ruby-identifier">doc</span>
-79:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 183</span>
+183:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setDoc</span>(<span class="ruby-identifier">keyword</span>, <span class="ruby-identifier">doc</span>)
+184:       <span class="ruby-ivar">@keyword</span> = <span class="ruby-identifier">keyword</span>
+185:       <span class="ruby-ivar">@doc</span> = <span class="ruby-identifier">doc</span>
+186:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1088,11 +1272,11 @@ Set the file and tag for the TJP code example.
 					<div class="method-source-code"
 						id="set-xample-source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 97</span>
- 97:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setExample</span>(<span class="ruby-identifier">file</span>, <span class="ruby-identifier">tag</span>)
- 98:       <span class="ruby-ivar">@exampleFile</span> = <span class="ruby-identifier">file</span>
- 99:       <span class="ruby-ivar">@exampleTag</span> = <span class="ruby-identifier">tag</span>
-100:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 204</span>
+204:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setExample</span>(<span class="ruby-identifier">file</span>, <span class="ruby-identifier">tag</span>)
+205:       <span class="ruby-ivar">@exampleFile</span> = <span class="ruby-identifier">file</span>
+206:       <span class="ruby-ivar">@exampleTag</span> = <span class="ruby-identifier">tag</span>
+207:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1125,10 +1309,10 @@ Restrict the syntax documentation to the first <tt>idx</tt> tokens.
 					<div class="method-source-code"
 						id="set-ast-yntax-oken-source">
 <pre>
-    <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 87</span>
-87:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setLastSyntaxToken</span>(<span class="ruby-identifier">idx</span>)
-88:       <span class="ruby-ivar">@lastSyntaxToken</span> = <span class="ruby-identifier">idx</span>
-89:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 194</span>
+194:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setLastSyntaxToken</span>(<span class="ruby-identifier">idx</span>)
+195:       <span class="ruby-ivar">@lastSyntaxToken</span> = <span class="ruby-identifier">idx</span>
+196:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1161,10 +1345,10 @@ Set the references to related patterns.
 					<div class="method-source-code"
 						id="set-ee-lso-source">
 <pre>
-    <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 92</span>
-92:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setSeeAlso</span>(<span class="ruby-identifier">also</span>)
-93:       <span class="ruby-ivar">@seeAlso</span> = <span class="ruby-identifier">also</span>
-94:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 199</span>
+199:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setSeeAlso</span>(<span class="ruby-identifier">also</span>)
+200:       <span class="ruby-ivar">@seeAlso</span> = <span class="ruby-identifier">also</span>
+201:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1197,10 +1381,10 @@ Returns true if the i-th token is a terminal symbol.
 					<div class="method-source-code"
 						id="terminal-ymbol--source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 139</span>
-139:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">terminalSymbol?</span>(<span class="ruby-identifier">i</span>)
-140:       <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">i</span>][<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">$$</span> <span class="ruby-operator">||</span> <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">i</span>][<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">__</span>
-141:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 245</span>
+245:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">terminalSymbol?</span>(<span class="ruby-identifier">i</span>)
+246:       <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">i</span>][<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">:variable</span> <span class="ruby-operator">||</span> <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">i</span>][<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">:literal</span>
+247:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1235,29 +1419,26 @@ The return value is an Array of [ token, pattern ] tuple.
 					<div class="method-source-code"
 						id="terminal-okens-source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 146</span>
-146:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">terminalTokens</span>(<span class="ruby-identifier">rules</span>, <span class="ruby-identifier">index</span> = <span class="ruby-value">0</span>)
-147:       <span class="ruby-comment cmt"># Terminal token start with an underscore or dollar character.</span>
-148:       <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">index</span>][<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">__</span>
-149:         <span class="ruby-keyword kw">return</span> [ [ <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">index</span>][<span class="ruby-value">1</span><span class="ruby-operator">..</span><span class="ruby-value">1</span>], <span class="ruby-keyword kw">self</span> ] ]
-150:       <span class="ruby-keyword kw">elsif</span> <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">index</span>][<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">$$</span>
-151:         <span class="ruby-keyword kw">return</span> []
-152:       <span class="ruby-keyword kw">elsif</span> <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">index</span>][<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">!!</span>
-153:         <span class="ruby-comment cmt"># Token starting with a bang reference another rule. We have to</span>
-154:         <span class="ruby-comment cmt"># continue the search at this rule. First, we get rid of the bang to</span>
-155:         <span class="ruby-comment cmt"># get the rule name.</span>
-156:         <span class="ruby-identifier">token</span> = <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">index</span>][<span class="ruby-value">1</span><span class="ruby-operator">..</span><span class="ruby-value">1</span>]
-157:         <span class="ruby-comment cmt"># Then find the rule</span>
-158:         <span class="ruby-identifier">rule</span> = <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">token</span>]
-159:         <span class="ruby-comment cmt"># The rule may only have a single pattern. If not, then this pattern</span>
-160:         <span class="ruby-comment cmt"># has no terminal token.</span>
-161:         <span class="ruby-identifier">tts</span> = []
-162:         <span class="ruby-identifier">rule</span>.<span class="ruby-identifier">patterns</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">p</span><span class="ruby-operator">|</span> <span class="ruby-identifier">tts</span> <span class="ruby-operator">+=</span> <span class="ruby-identifier">p</span>.<span class="ruby-identifier">terminalTokens</span>(<span class="ruby-identifier">rules</span>, <span class="ruby-value">0</span>) }
-163:         <span class="ruby-keyword kw">return</span> <span class="ruby-identifier">tts</span>
-164:       <span class="ruby-keyword kw">else</span>
-165:         <span class="ruby-identifier">raise</span> <span class="ruby-node">&quot;Unexpected token #{tokens[index]}&quot;</span>
-166:       <span class="ruby-keyword kw">end</span>
-167:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 252</span>
+252:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">terminalTokens</span>(<span class="ruby-identifier">rules</span>, <span class="ruby-identifier">index</span> = <span class="ruby-value">0</span>)
+253:       <span class="ruby-identifier">type</span>, <span class="ruby-identifier">name</span> = <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">index</span>]
+254:       <span class="ruby-comment cmt"># Terminal token start with an underscore or dollar character.</span>
+255:       <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:literal</span>
+256:         <span class="ruby-keyword kw">return</span> [ [ <span class="ruby-identifier">name</span>, <span class="ruby-keyword kw">self</span> ] ]
+257:       <span class="ruby-keyword kw">elsif</span> <span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:variable</span>
+258:         <span class="ruby-keyword kw">return</span> []
+259:       <span class="ruby-keyword kw">elsif</span> <span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:reference</span>
+260:         <span class="ruby-comment cmt"># We have to continue the search at this rule.</span>
+261:         <span class="ruby-identifier">rule</span> = <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">name</span>]
+262:         <span class="ruby-comment cmt"># The rule may only have a single pattern. If not, then this pattern</span>
+263:         <span class="ruby-comment cmt"># has no terminal token.</span>
+264:         <span class="ruby-identifier">tts</span> = []
+265:         <span class="ruby-identifier">rule</span>.<span class="ruby-identifier">patterns</span>.<span class="ruby-identifier">each</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">p</span><span class="ruby-operator">|</span> <span class="ruby-identifier">tts</span> <span class="ruby-operator">+=</span> <span class="ruby-identifier">p</span>.<span class="ruby-identifier">terminalTokens</span>(<span class="ruby-identifier">rules</span>, <span class="ruby-value">0</span>) }
+266:         <span class="ruby-keyword kw">return</span> <span class="ruby-identifier">tts</span>
+267:       <span class="ruby-keyword kw">else</span>
+268:         <span class="ruby-identifier">raise</span> <span class="ruby-node">&quot;Unexpected token #{type} #{name}&quot;</span>
+269:       <span class="ruby-keyword kw">end</span>
+270:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1281,17 +1462,36 @@ The return value is an Array of [ token, pattern ] tuple.
 
 				<div class="method-description">
 					
-					
+					<p>
+Generate a text form of the pattern. This is similar to the syntax in the
+original syntax description.
+</p>
 					
 
 					
 					<div class="method-source-code"
 						id="to-s-source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 251</span>
-251:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">to_s</span>
-252:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">join</span>(<span class="ruby-value str">' '</span>)
-253:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 354</span>
+354:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">to_s</span>
+355:       <span class="ruby-identifier">str</span> = <span class="ruby-value str">&quot;&quot;</span>
+356:       <span class="ruby-ivar">@tokens</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">type</span>, <span class="ruby-identifier">name</span><span class="ruby-operator">|</span>
+357:         <span class="ruby-keyword kw">case</span> <span class="ruby-identifier">type</span>
+358:         <span class="ruby-keyword kw">when</span> <span class="ruby-value">:reference</span>
+359:           <span class="ruby-identifier">str</span> <span class="ruby-operator">+=</span> <span class="ruby-node">&quot;!#{name} &quot;</span>
+360:         <span class="ruby-keyword kw">when</span> <span class="ruby-value">:variable</span>
+361:           <span class="ruby-identifier">str</span> <span class="ruby-operator">+=</span> <span class="ruby-node">&quot;$#{name } &quot;</span>
+362:         <span class="ruby-keyword kw">when</span> <span class="ruby-value">:literal</span>
+363:           <span class="ruby-identifier">str</span> <span class="ruby-operator">+=</span> <span class="ruby-node">&quot;#{name} &quot;</span>
+364:         <span class="ruby-keyword kw">when</span> <span class="ruby-value">:eof</span>
+365:           <span class="ruby-identifier">str</span> <span class="ruby-operator">+=</span> <span class="ruby-value str">&quot;. &quot;</span>
+366:         <span class="ruby-keyword kw">else</span>
+367:           <span class="ruby-identifier">raise</span> <span class="ruby-node">&quot;Unknown type #{type}&quot;</span>
+368:         <span class="ruby-keyword kw">end</span>
+369:       <span class="ruby-keyword kw">end</span>
+370: 
+371:       <span class="ruby-identifier">str</span>
+372:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1326,10 +1526,10 @@ the wrapper function that sets up the stack.
 					<div class="method-source-code"
 						id="to-syntax-source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 172</span>
-172:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">to_syntax</span>(<span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-identifier">skip</span> = <span class="ruby-value">0</span>)
-173:       <span class="ruby-identifier">to_syntax_r</span>({}, <span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-identifier">skip</span>)
-174:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 275</span>
+275:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">to_syntax</span>(<span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-identifier">skip</span> = <span class="ruby-value">0</span>)
+276:       <span class="ruby-identifier">to_syntax_r</span>({}, <span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-identifier">skip</span>)
+277:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1353,88 +1553,87 @@ the wrapper function that sets up the stack.
 
 				<div class="method-description">
 					
-					
+					<p>
+Generate a syntax description for this pattern.
+</p>
 					
 
 					
 					<div class="method-source-code"
 						id="to-syntax-r-source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 176</span>
-176:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">to_syntax_r</span>(<span class="ruby-identifier">stack</span>, <span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-identifier">skip</span>)
-177:       <span class="ruby-comment cmt"># If we find ourself on the stack we hit a recursive pattern. This is used</span>
-178:       <span class="ruby-comment cmt"># in repetitions.</span>
-179:       <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">stack</span>[<span class="ruby-keyword kw">self</span>]
-180:         <span class="ruby-keyword kw">return</span> <span class="ruby-value str">'[, ... ]'</span>
-181:       <span class="ruby-keyword kw">end</span>
-182: 
-183:       <span class="ruby-comment cmt"># &quot;Push&quot; us on the stack.</span>
-184:       <span class="ruby-identifier">stack</span>[<span class="ruby-keyword kw">self</span>] = <span class="ruby-keyword kw">true</span>
-185: 
-186:       <span class="ruby-identifier">str</span> = <span class="ruby-value str">''</span>
-187:       <span class="ruby-identifier">first</span> = <span class="ruby-keyword kw">true</span>
-188:       <span class="ruby-comment cmt"># Analyze the tokens of the pattern skipping the first 'skip' tokens.</span>
-189:       <span class="ruby-identifier">skip</span>.<span class="ruby-identifier">upto</span>(<span class="ruby-ivar">@lastSyntaxToken</span>) <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">i</span><span class="ruby-operator">|</span>
-190:         <span class="ruby-identifier">token</span> = <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">i</span>]
-191:         <span class="ruby-comment cmt"># If the first token is a _{ the pattern describes optional attributes.</span>
-192:         <span class="ruby-comment cmt"># They are represented by a standard idiom.</span>
-193:         <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">first</span>
-194:           <span class="ruby-identifier">first</span> = <span class="ruby-keyword kw">false</span>
-195:           <span class="ruby-keyword kw">return</span> <span class="ruby-value str">'{ &lt;attributes&gt; }'</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">token</span> <span class="ruby-operator">==</span> <span class="ruby-value str">'_{'</span>
-196:         <span class="ruby-keyword kw">else</span>
-197:           <span class="ruby-comment cmt"># Separate the syntax elemens by a whitespace.</span>
-198:           <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-value str">' '</span>
-199:         <span class="ruby-keyword kw">end</span>
-200: 
-201:         <span class="ruby-identifier">typeId</span> = <span class="ruby-identifier">token</span>[<span class="ruby-value">0</span>]
-202:         <span class="ruby-identifier">token</span> = <span class="ruby-identifier">token</span>.<span class="ruby-identifier">slice</span>(<span class="ruby-value">1</span>, <span class="ruby-identifier">token</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">-</span> <span class="ruby-value">1</span>)
-203: 
-204:         <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@args</span>[<span class="ruby-identifier">i</span>]
-205:           <span class="ruby-comment cmt"># The argument is documented in the syntax definition. We copy the</span>
-206:           <span class="ruby-comment cmt"># entry as we need to modify it.</span>
-207:           <span class="ruby-identifier">argDoc</span> = <span class="ruby-ivar">@args</span>[<span class="ruby-identifier">i</span>].<span class="ruby-identifier">dup</span>
-208: 
-209:           <span class="ruby-comment cmt"># A documented argument without a name is a terminal token. We use the</span>
-210:           <span class="ruby-comment cmt"># terminal symbol as name.</span>
-211:           <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@args</span>[<span class="ruby-identifier">i</span>].<span class="ruby-identifier">name</span>.<span class="ruby-identifier">nil?</span>
-212:             <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot;#{token}&quot;</span>
-213:             <span class="ruby-identifier">argDoc</span>.<span class="ruby-identifier">name</span> = <span class="ruby-identifier">token</span>
-214:           <span class="ruby-keyword kw">else</span>
-215:             <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot;&lt;#{@args[i].name}&gt;&quot;</span>
-216:           <span class="ruby-keyword kw">end</span>
-217:           <span class="ruby-identifier">addArgDoc</span>(<span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">argDoc</span>)
-218: 
-219:           <span class="ruby-comment cmt"># Documented arguments don't have the type set yet. Use the token</span>
-220:           <span class="ruby-comment cmt"># value for that.</span>
-221:           <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">typeId</span> <span class="ruby-operator">==</span> <span class="ruby-value">$$</span>
-222:             <span class="ruby-identifier">argDoc</span>.<span class="ruby-identifier">typeSpec</span> = <span class="ruby-node">&quot;&lt;#{token}&gt;&quot;</span>
-223:           <span class="ruby-keyword kw">end</span>
-224:         <span class="ruby-keyword kw">else</span>
-225:           <span class="ruby-comment cmt"># Undocumented tokens are recursively expanded.</span>
-226:           <span class="ruby-keyword kw">case</span> <span class="ruby-identifier">typeId</span>
-227:           <span class="ruby-keyword kw">when</span> <span class="ruby-value">__</span>
-228:             <span class="ruby-comment cmt"># Literals are shown as such.</span>
-229:             <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">token</span>
-230:           <span class="ruby-keyword kw">when</span> <span class="ruby-value">$$</span>
-231:             <span class="ruby-comment cmt"># Variables are enclosed by angle brackets.</span>
-232:             <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-value str">'&lt;'</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">token</span> <span class="ruby-operator">+</span> <span class="ruby-value str">'&gt;'</span>
-233:           <span class="ruby-keyword kw">when</span> <span class="ruby-value">!!</span>
-234:             <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">token</span>].<span class="ruby-identifier">patterns</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">==</span> <span class="ruby-value">1</span> <span class="ruby-operator">&amp;&amp;</span>
-235:                <span class="ruby-operator">!</span><span class="ruby-identifier">rules</span>[<span class="ruby-identifier">token</span>].<span class="ruby-identifier">patterns</span>[<span class="ruby-value">0</span>].<span class="ruby-identifier">doc</span>.<span class="ruby-identifier">nil?</span>
-236:               <span class="ruby-identifier">addArgDoc</span>(<span class="ruby-identifier">argDocs</span>, <span class="ruby-constant">TokenDoc</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">rules</span>[<span class="ruby-identifier">token</span>].<span class="ruby-identifier">patterns</span>[<span class="ruby-value">0</span>].<span class="ruby-identifier">keyword</span>,
-237:                                               <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">token</span>].<span class="ruby-identifier">patterns</span>[<span class="ruby-value">0</span>]))
-238:               <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-value str">'&lt;'</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">token</span>].<span class="ruby-identifier">patterns</span>[<span class="ruby-value">0</span>].<span class="ruby-identifier">keyword</span> <span class="ruby-operator">+</span> <span class="ruby-value str">'&gt;'</span>
-239:             <span class="ruby-keyword kw">else</span>
-240:               <span class="ruby-comment cmt"># References are followed recursively.</span>
-241:               <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">token</span>].<span class="ruby-identifier">to_syntax</span>(<span class="ruby-identifier">stack</span>, <span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-value">0</span>)
-242:             <span class="ruby-keyword kw">end</span>
-243:           <span class="ruby-keyword kw">end</span>
-244:         <span class="ruby-keyword kw">end</span>
-245:       <span class="ruby-keyword kw">end</span>
-246:       <span class="ruby-comment cmt"># Remove us from the &quot;stack&quot; again.</span>
-247:       <span class="ruby-identifier">stack</span>.<span class="ruby-identifier">delete</span>(<span class="ruby-keyword kw">self</span>)
-248:       <span class="ruby-identifier">str</span>
-249:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 280</span>
+280:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">to_syntax_r</span>(<span class="ruby-identifier">stack</span>, <span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-identifier">skip</span>)
+281:       <span class="ruby-comment cmt"># If we find ourself on the stack we hit a recursive pattern. This is used</span>
+282:       <span class="ruby-comment cmt"># in repetitions.</span>
+283:       <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">stack</span>[<span class="ruby-keyword kw">self</span>]
+284:         <span class="ruby-keyword kw">return</span> <span class="ruby-value str">'[, ... ]'</span>
+285:       <span class="ruby-keyword kw">end</span>
+286: 
+287:       <span class="ruby-comment cmt"># &quot;Push&quot; us on the stack.</span>
+288:       <span class="ruby-identifier">stack</span>[<span class="ruby-keyword kw">self</span>] = <span class="ruby-keyword kw">true</span>
+289: 
+290:       <span class="ruby-identifier">str</span> = <span class="ruby-value str">''</span>
+291:       <span class="ruby-identifier">first</span> = <span class="ruby-keyword kw">true</span>
+292:       <span class="ruby-comment cmt"># Analyze the tokens of the pattern skipping the first 'skip' tokens.</span>
+293:       <span class="ruby-identifier">skip</span>.<span class="ruby-identifier">upto</span>(<span class="ruby-ivar">@lastSyntaxToken</span>) <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">i</span><span class="ruby-operator">|</span>
+294:         <span class="ruby-identifier">type</span>, <span class="ruby-identifier">name</span> = <span class="ruby-ivar">@tokens</span>[<span class="ruby-identifier">i</span>]
+295:         <span class="ruby-comment cmt"># If the first token is a _{ the pattern describes optional attributes.</span>
+296:         <span class="ruby-comment cmt"># They are represented by a standard idiom.</span>
+297:         <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">first</span>
+298:           <span class="ruby-identifier">first</span> = <span class="ruby-keyword kw">false</span>
+299:           <span class="ruby-keyword kw">return</span> <span class="ruby-value str">'{ &lt;attributes&gt; }'</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">name</span> <span class="ruby-operator">==</span> <span class="ruby-value str">'{'</span>
+300:         <span class="ruby-keyword kw">else</span>
+301:           <span class="ruby-comment cmt"># Separate the syntax elemens by a whitespace.</span>
+302:           <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-value str">' '</span>
+303:         <span class="ruby-keyword kw">end</span>
+304: 
+305:         <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@args</span>[<span class="ruby-identifier">i</span>]
+306:           <span class="ruby-comment cmt"># The argument is documented in the syntax definition. We copy the</span>
+307:           <span class="ruby-comment cmt"># entry as we need to modify it.</span>
+308:           <span class="ruby-identifier">argDoc</span> = <span class="ruby-ivar">@args</span>[<span class="ruby-identifier">i</span>].<span class="ruby-identifier">dup</span>
+309: 
+310:           <span class="ruby-comment cmt"># A documented argument without a name is a terminal token. We use the</span>
+311:           <span class="ruby-comment cmt"># terminal symbol as name.</span>
+312:           <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@args</span>[<span class="ruby-identifier">i</span>].<span class="ruby-identifier">name</span>.<span class="ruby-identifier">nil?</span>
+313:             <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot;#{name}&quot;</span>
+314:             <span class="ruby-identifier">argDoc</span>.<span class="ruby-identifier">name</span> = <span class="ruby-identifier">name</span>
+315:           <span class="ruby-keyword kw">else</span>
+316:             <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot;&lt;#{@args[i].name}&gt;&quot;</span>
+317:           <span class="ruby-keyword kw">end</span>
+318:           <span class="ruby-identifier">addArgDoc</span>(<span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">argDoc</span>)
+319: 
+320:           <span class="ruby-comment cmt"># Documented arguments don't have the type set yet. Use the token</span>
+321:           <span class="ruby-comment cmt"># value for that.</span>
+322:           <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:variable</span>
+323:             <span class="ruby-identifier">argDoc</span>.<span class="ruby-identifier">typeSpec</span> = <span class="ruby-node">&quot;&lt;#{name}&gt;&quot;</span>
+324:           <span class="ruby-keyword kw">end</span>
+325:         <span class="ruby-keyword kw">else</span>
+326:           <span class="ruby-comment cmt"># Undocumented tokens are recursively expanded.</span>
+327:           <span class="ruby-keyword kw">case</span> <span class="ruby-identifier">type</span>
+328:           <span class="ruby-keyword kw">when</span> <span class="ruby-value">:literal</span>
+329:             <span class="ruby-comment cmt"># Literals are shown as such.</span>
+330:             <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">name</span>.<span class="ruby-identifier">to_s</span>
+331:           <span class="ruby-keyword kw">when</span> <span class="ruby-value">:variable</span>
+332:             <span class="ruby-comment cmt"># Variables are enclosed by angle brackets.</span>
+333:             <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot;&lt;#{name}&gt;&quot;</span>
+334:           <span class="ruby-keyword kw">when</span> <span class="ruby-value">:reference</span>
+335:             <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">name</span>].<span class="ruby-identifier">patterns</span>.<span class="ruby-identifier">length</span> <span class="ruby-operator">==</span> <span class="ruby-value">1</span> <span class="ruby-operator">&amp;&amp;</span>
+336:                <span class="ruby-operator">!</span><span class="ruby-identifier">rules</span>[<span class="ruby-identifier">name</span>].<span class="ruby-identifier">patterns</span>[<span class="ruby-value">0</span>].<span class="ruby-identifier">doc</span>.<span class="ruby-identifier">nil?</span>
+337:               <span class="ruby-identifier">addArgDoc</span>(<span class="ruby-identifier">argDocs</span>, <span class="ruby-constant">TokenDoc</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">rules</span>[<span class="ruby-identifier">name</span>].<span class="ruby-identifier">patterns</span>[<span class="ruby-value">0</span>].<span class="ruby-identifier">keyword</span>,
+338:                                               <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">name</span>].<span class="ruby-identifier">patterns</span>[<span class="ruby-value">0</span>]))
+339:               <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-value str">'&lt;'</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">name</span>].<span class="ruby-identifier">patterns</span>[<span class="ruby-value">0</span>].<span class="ruby-identifier">keyword</span> <span class="ruby-operator">+</span> <span class="ruby-value str">'&gt;'</span>
+340:             <span class="ruby-keyword kw">else</span>
+341:               <span class="ruby-comment cmt"># References are followed recursively.</span>
+342:               <span class="ruby-identifier">str</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">name</span>].<span class="ruby-identifier">to_syntax</span>(<span class="ruby-identifier">stack</span>, <span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">rules</span>, <span class="ruby-value">0</span>)
+343:             <span class="ruby-keyword kw">end</span>
+344:           <span class="ruby-keyword kw">end</span>
+345:         <span class="ruby-keyword kw">end</span>
+346:       <span class="ruby-keyword kw">end</span>
+347:       <span class="ruby-comment cmt"># Remove us from the &quot;stack&quot; again.</span>
+348:       <span class="ruby-identifier">stack</span>.<span class="ruby-identifier">delete</span>(<span class="ruby-keyword kw">self</span>)
+349:       <span class="ruby-identifier">str</span>
+350:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>
@@ -1471,14 +1670,14 @@ the wrapper function that sets up the stack.
 					<div class="method-source-code"
 						id="add-rg-oc-source">
 <pre>
-     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 257</span>
-257:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">addArgDoc</span>(<span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">argDoc</span>)
-258:       <span class="ruby-identifier">raise</span> <span class="ruby-value str">'Error'</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">argDoc</span>.<span class="ruby-identifier">name</span>.<span class="ruby-identifier">nil?</span>
-259:       <span class="ruby-identifier">argDocs</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">ad</span><span class="ruby-operator">|</span>
-260:         <span class="ruby-keyword kw">return</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">ad</span>.<span class="ruby-identifier">name</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">argDoc</span>.<span class="ruby-identifier">name</span>
-261:       <span class="ruby-keyword kw">end</span>
-262:       <span class="ruby-identifier">argDocs</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">argDoc</span>
-263:     <span class="ruby-keyword kw">end</span></pre>
+     <span class="ruby-comment cmt"># File lib/TextParser/Pattern.rb, line 376</span>
+376:     <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">addArgDoc</span>(<span class="ruby-identifier">argDocs</span>, <span class="ruby-identifier">argDoc</span>)
+377:       <span class="ruby-identifier">raise</span> <span class="ruby-value str">'Error'</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">argDoc</span>.<span class="ruby-identifier">name</span>.<span class="ruby-identifier">nil?</span>
+378:       <span class="ruby-identifier">argDocs</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">ad</span><span class="ruby-operator">|</span>
+379:         <span class="ruby-keyword kw">return</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">ad</span>.<span class="ruby-identifier">name</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">argDoc</span>.<span class="ruby-identifier">name</span>
+380:       <span class="ruby-keyword kw">end</span>
+381:       <span class="ruby-identifier">argDocs</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">argDoc</span>
+382:     <span class="ruby-keyword kw">end</span></pre>
 					</div>
 					
 				</div>