tokn 0.0.5 → 0.0.6

data/lib/tokn/dfa.rb

@@ -2,195 +2,220 @@ require 'json'
 require_relative 'tools'
 req('code_set state')
 
-
-# A DFA for tokenizing; includes pointer to a start state, and
-# a list of token names
-#
-class DFA
-  
-  include Tokn
-
-  # Compile a Tokenizer DFA from a token definition script.
-  # If persistPath is not null, it first checks if the file exists and
-  # if so, assumes it contains (in JSON form) a previously compiled 
-  # DFA matching this script, and reads the DFA from it.  
-  # Second, if no such file exists, it writes the DFA to it after compilation.
+module Tokn
+   
+  # A DFA for tokenizing; includes pointer to a start state, and
+  # a list of token names
   #
-  def self.dfa_from_script(script, persistPath = nil)
+  class DFA
     
-    if persistPath and File.exist?(persistPath)
-      return extractDFA(readTextFile(persistPath))
-    end
-    
-    req('token_defn_parser')
+    include ToknInternal
     
-    td = TokenDefParser.new(script)
-    dfa = td.dfa
+    # Compile a Tokenizer DFA from a token definition script.
+    # If persistPath is not null, it first checks if the file exists and
+    # if so, assumes it contains (in JSON form) a previously compiled 
+    # DFA matching this script, and reads the DFA from it.  
+    # Second, if no such file exists, it writes the DFA to it after compilation.
+    #
+    def self.from_script(script, persistPath = nil)
+      
+      if persistPath and File.exist?(persistPath)
+        return extractDFA(readTextFile(persistPath))
+      end
+      
+      req('token_defn_parser')
+      
+      td = TokenDefParser.new(script)
+      dfa = td.dfa
+      
+      if persistPath
+        writeTextFile(persistPath, dfa.serialize())
+      end
     
-    if persistPath
-      writeTextFile(persistPath, dfa.serialize())
+      dfa  
     end
-  
-    dfa  
-  end
-  
-  # Similar to dfa_from_script, but reads the script into memory from 
-  # the file at scriptPath.
-  #
-  def self.dfa_from_script_file(scriptPath, persistPath = nil)
-    self.dfa_from_script(readTextFile(scriptPath), persistPath)  
-  end
-  
-  # Compile a Tokenizer DFA from a text file (that contains a
-  # JSON string)
-  #
-  def self.dfa_from_file(path)
-    dfa_from_json(readTextFile(path))
-  end
-  
-  # Compile a Tokenizer DFA from a JSON string
-  #
-  def self.dfa_from_json(jsonStr)
-    db = false
     
-    !db|| pr("\n\nextractDFA %s...\n",jsonStr)
-    
-    h = JSON.parse(jsonStr)
-    
-    tNames = h["tokens"]
-    stateInfo = h["states"]
-    
-    !db|| pr("tokens=%s\n",d(tNames))
-    !db|| pr("stateInfo=\n%s\n",d(stateInfo))
-    
-    st = []
-    stateInfo.each_with_index do |(key,val),i|
-      !db|| pr(" creating new state, id=%d\n",i)
-      st.push(State.new(i))
+    # Similar to from_script, but reads the script into memory from 
+    # the file at scriptPath.
+    #
+    def self.from_script_file(scriptPath, persistPath = nil)
+      self.from_script(readTextFile(scriptPath), persistPath)  
     end
     
-    st.each do |s|
-      !db|| pr("proc state %s\n",d(s))
-      
-      finalState, edgeList = stateInfo[s.id]
-      s.finalState = finalState
-      edgeList.each do |edge|
-        label,destState = edge
-        cr = CodeSet.new()
-        cr.setArray(label)
-        s.addEdge(cr, st[destState])
-      end
+    # Compile a Tokenizer DFA from a text file (that contains a
+    # JSON string)
+    #
+    def self.from_file(path)
+      from_json(readTextFile(path))
     end
     
-    DFA.new(tNames, st[0])
-  
-  end
-  
-  attr_reader :startState, :tokenNames
-
-  # Construct a DFA, given a list of token names and a starting state.
-  #
-  def initialize(tokenNameList, startState)
-    @tokenNames = tokenNameList
-    @startState = startState
-  end
-  
-  # Determine the name of a token, given its id.
-  # Returns <UNKNOWN> if its id is UNKNOWN_TOKEN, or <EOF> if
-  # the tokenId is nil.  Otherwise, assumes tokenId is 0..n, where
-  # n is the number of token names in the DFA.
-  #
-  def tokenName(tokenId)
-    if !tokenId
-      nm = "<EOF>"
-    elsif tokenId == UNKNOWN_TOKEN
-      nm = "<UNKNOWN>"
-    else
-      if tokenId < 0 || tokenId >= tokenNames.size
-        raise IndexError, "No such token id: "+tokenId.to_s
+    # Compile a Tokenizer DFA from a JSON string
+    #
+    def self.from_json(jsonStr)
+      db = false
+      
+      !db|| pr("\n\nextractDFA %s...\n",jsonStr)
+      
+      h = JSON.parse(jsonStr)
+      
+      version = h["version"]
+      
+      if !version || version.floor != VERSION.floor 
+        raise ArgumentError, 
+           "Bad or missing version number: "+version.to_s+", expected "+VERSION.to_s
       end
-      nm = tokenNames[tokenId]
-    end 
-    nm 
-  end
-  
-  # Serialize this DFA to a JSON string.
-  # The DFA in JSON form has this structure:
-  #
-  #  {
-  #    "tokens" => array of token names (strings)
-  #    "states" => array of states, ordered by id (0,1,..)
-  #  }
-  #
-  # Each state has this format:
-  #  [ finalState (boolean),
-  #   [edge0, edge1, ...]
-  #  ]
-  #
-  # Edge:
-  #  [label, destination id (integer)]
-  #
-  # Labels are arrays of integers, exactly the structure of
-  # a CodeSet array.
-  #
-  def serialize 
-    
-    h = {}
-    
-    h["tokens"] =  tokenNames
-    
-    stateSet,_,_ = startState.reachableStates
+      
+      tNames = h["tokens"]
+      stateInfo = h["states"]
+      
+      !db|| pr("tokens=%s\n",d(tNames))
+      !db|| pr("stateInfo=\n%s\n",d(stateInfo))
+      
+      st = []
+      stateInfo.each_with_index do |(key,val),i|
+        !db|| pr(" creating new state, id=%d\n",i)
+        st.push(State.new(i))
+      end
+      
+      st.each do |s|
+        !db|| pr("proc state %s\n",d(s))
+        
+        finalState, edgeList = stateInfo[s.id]
+        s.finalState = finalState
+        edgeList.each do |edge|
+          label,destState = edge
+          cr = CodeSet.new()
+          cr.setArray(label)
+          s.addEdge(cr, st[destState])
+        end
+      end
+      
+      DFA.new(tNames, st[0])
     
-    idToStateMap = {}
-    stateSet.each do |st|
-      idToStateMap[st.id] = st  
     end
     
-    stateList = []
-    
-    nextId = 0
-    idToStateMap.each_pair do |id, st|
-      if nextId != id
-        raise ArgumentError, "unexpected state ids"
+    attr_reader :startState, :tokenNames
+  
+    # Construct a DFA, given a list of token names and a starting state.
+    #
+    def initialize(tokenNameList, startState)
+      
+      if (startState.id != 0)
+        raise ArgumentError, "Start state id must be zero"
+      end
+      
+      @tokenNames = tokenNameList
+      @startState = startState
+      @tokenIdMap = {}
+      @tokenNames.each_with_index do |name, i|
+        @tokenIdMap[name] = i
       end
-      nextId += 1
       
-      stateList.push(st)
     end
     
-    if stateList.size == 0
-      raise ArgumentError, "bad states"
+    # Determine the name of a token, given its id.
+    # Returns <UNKNOWN> if its id is UNKNOWN_TOKEN, or <EOF> if
+    # the tokenId is nil.  Otherwise, assumes tokenId is 0 ... n-1, where
+    # n is the number of token names in the DFA.
+    #
+    def tokenName(tokenId)
+      if !tokenId
+        nm = "<EOF>"
+      elsif tokenId == UNKNOWN_TOKEN
+        nm = "<UNKNOWN>"
+      else
+        if tokenId < 0 || tokenId >= tokenNames.size
+          raise IndexError, "No such token id: "+tokenId.to_s
+        end
+        nm = tokenNames[tokenId]
+      end 
+      nm 
     end
     
-    if stateList[0] != startState
-      raise ArgumentError, "bad start state"
+    # Get id of token given its name
+    # @param tokenName name of token
+    # @return nil if there is no token with that name
+    #
+    def tokenId(tokenName)
+      @tokenIdMap[tokenName]
     end
     
-    stateInfo = []
-    stateList.each do |st|
-        stateInfo.push(stateToList(st))
+    # Serialize this DFA to a JSON string.
+    # The DFA in JSON form has this structure:
+    #
+    #  {
+    #    "version" => version number (float)
+    #    "tokens" => array of token names (strings)
+    #    "states" => array of states, ordered by id (0,1,..)
+    #  }
+    #
+    # Each state has this format:
+    #  [ finalState (boolean),
+    #   [edge0, edge1, ...]
+    #  ]
+    #
+    # Edge:
+    #  [label, destination id (integer)]
+    #
+    # Labels are arrays of integers, exactly the structure of
+    # a CodeSet array.
+    #
+    def serialize 
+      
+      h = {"version"=>VERSION, "tokens"=>tokenNames}
+      
+      
+      stateSet,_,_ = startState.reachableStates
+      
+      idToStateMap = {}
+      stateSet.each{ |st| idToStateMap[st.id] = st }
+      
+      stateList = []
+      
+      nextId = 0
+      idToStateMap.each_pair do |id, st|
+        if nextId != id
+          raise ArgumentError, "unexpected state ids"
+        end
+        nextId += 1
+        
+        stateList.push(st)
+      end
+      
+      if stateList.size == 0
+        raise ArgumentError, "bad states"
+      end
+      
+      if stateList[0] != startState
+        raise ArgumentError, "bad start state"
+      end
+      
+      stateInfo = []
+      stateList.each do |st|
+          stateInfo.push(stateToList(st))
+      end
+      h["states"] = stateInfo 
+      
+      JSON.generate(h)
     end
-    h["states"] = stateInfo 
-    
-    JSON.generate(h)
-  end
-
-  private
   
-  def stateToList(state)
-    list = []
+    private
+    
+    VERSION = 1.0
     
-    list.push(state.finalState?)
-    ed = []
-    state.edges.each do |lbl, dest|
-      edInfo = [lbl.array, dest.id]
-      ed.push(edInfo)
+    def stateToList(state)
+      list = [state.finalState?]
+      ed = []
+      state.edges.each do |lbl, dest|
+        edInfo = [lbl.array, dest.id]
+        ed.push(edInfo)
+      end
+      list.push(ed)
+      
+      list
     end
-    list.push(ed)
     
-    list
   end
   
-end
-
+end  # module Tokn
 

data/lib/tokn/dfa_builder.rb

@@ -1,261 +1,259 @@
 require_relative 'tools'
-req('tokn_const code_set state range_partition reg_parse')
+# req('tokn_const code_set state range_partition reg_parse')
+req('range_partition reg_parse')
 
-# Converts NFAs (nondeterministic, finite state automata) to
-# minimal DFAs.
-#
-# Performs the subset construction algorithm described in 
-# (among other placess) http://en.wikipedia.org/wiki/Powerset_construction
-#
-# Also implements an innovative algorithm to partition a set of 
-# edge labels into a set that has the property that no two elements
-# have overlapping regions.  This allows us to perform the subset construction
-# (and closure operations) efficiently while supporting large possible character
-# sets (e.g., unicode, which ranges from 0..0x10ffff.  See RangePartition.rb
-# for more details.
-#
-class DFABuilder
-
-  include Tokn
+module ToknInternal
   
-
-  # Convert an NFA to a DFA.
-  # 
-  # @param startState the start state of the NFA
-  # @param db if true, generates PDF files for debug purposes, showing various 
-  #    steps of the procedure
+  # Converts NFAs (nondeterministic, finite state automata) to
+  # minimal DFAs.
   #
-  def self.nfa_to_dfa(startState, db = false)
-   
-    !db || startState.generatePDF("original_nfa")
-    
-    # Reverse this NFA, convert to DFA, then
-    # reverse it, and convert it again.  Apparently this
-    # produces a minimal DFA.
-    
-    rev = startState.reverseNFA()
-    !db || rev.generatePDF("reversed_nfa")
-    
-    bld = DFABuilder.new(rev)
-    dfa = bld.build(true, false)  # partition, but don't normalize
-    
-    !db || dfa.generatePDF("reversed_dfa")
-    
-    rev2 = dfa.reverseNFA()
-    bld = DFABuilder.new(rev2)
-    
-    # Don't regenerate the partition; it is still valid
-    # for this second build process
-    #
-    dfa = bld.build(false, true) # don't partition, but do normalize
-    
-    # If there are edges that contain more than one token identifier,
-    # remove all but the first (i.e. the one with the highest token id)
-    
-    stSet, _, _ = dfa.reachableStates
-    stSet.each do |s|
-      s.edges.each do |lbl, dest|
-        a = lbl.array
-        if !a.size
-          next
-        end
-        
-        primeId = a[0]
-        
-        if primeId >= EPSILON-1
-          next
-        end
-        
-        lbl.difference!(CodeSet.new(primeId+1, EPSILON))
-      end
-    end
-    
-    !db || dfa.generatePDF("minimal_dfa")
-    
-    dfa
-  end
-  
-    
-  
-  # Constructs a builder object
+  # Performs the subset construction algorithm described in 
+  # (among other placess) http://en.wikipedia.org/wiki/Powerset_construction
   #
-  def initialize(nfaStartState)
-    @nextId = 0
-    @nfaStart = nfaStartState
-    
-    # Build a map of nfa state ids => nfa states
-    @nfaStateMap = {}
-    nfas, _, _ = @nfaStart.reachableStates
-    nfas.each {|s| @nfaStateMap[s.id] = s}
-    
-    # Initialize an array of nfa state lists, indexed by dfa state id
-    @nfaStateLists = []
-    
-    # Map of existing DFA states; key is array of NFA state ids
-    @dfaStateMap = {}
-  end
-
-  # Perform the build algorithm
-  #
-  # @param partition if true, partitions the edge labels into disjoint code sets
-  # @param normalize if true, normalizes the states afterward
+  # Also implements an innovative algorithm to partition a set of 
+  # edge labels into a set that has the property that no two elements
+  # have overlapping regions.  This allows us to perform the subset construction
+  # (and closure operations) efficiently while supporting large possible character
+  # sets (e.g., unicode, which ranges from 0..0x10ffff.  See RangePartition.rb
+  # for more details.
   #
-  def build(partition = true, normalize = true)
-    db = false
-    
-    !partition || partitionEdges(@nfaStart)
-    
-    iset = Set.new
-    iset.add(@nfaStart)
-    epsClosure(iset)
-    
-    @dfaStart,_ = createDFAState(stateSetToIdArray(iset))
-    
-    markedStates = Set.new
-    
-    unmarked = [@dfaStart]
-
-    until unmarked.empty? 
-      dfaState  = unmarked.pop
+  class DFABuilder
+   
+    # Convert an NFA to a DFA.
+    # 
+    # @param startState the start state of the NFA
+    # @param db if true, generates PDF files for debug purposes, showing various 
+    #    steps of the procedure
+    #
+    def self.nfa_to_dfa(startState, db = false)
+     
+      !db || startState.generatePDF("original_nfa")
+      
+      # Reverse this NFA, convert to DFA, then
+      # reverse it, and convert it again.  Apparently this
+      # produces a minimal DFA.
+      
+      rev = startState.reverseNFA()
+      !db || rev.generatePDF("reversed_nfa")
+      
+      bld = DFABuilder.new(rev)
+      dfa = bld.build(true, false)  # partition, but don't normalize
+      
+      !db || dfa.generatePDF("reversed_dfa")
       
-      nfaIds = @nfaStateLists[dfaState.id]
+      rev2 = dfa.reverseNFA()
+      bld = DFABuilder.new(rev2)
       
-      # map of CodeSet => set of NFA states
-      moveMap = {}
+      # Don't regenerate the partition; it is still valid
+      # for this second build process
+      #
+      dfa = bld.build(false, true) # don't partition, but do normalize
       
-      nfaIds.each do |nfaId| 
-        nfaState = @nfaStateMap[nfaId]
-        nfaState.edges.each do |lbl,dest| 
-          if lbl.array[0] == EPSILON
+      # If there are edges that contain more than one token identifier,
+      # remove all but the first (i.e. the one with the highest token id)
+      
+      stSet, _, _ = dfa.reachableStates
+      stSet.each do |s|
+        s.edges.each do |lbl, dest|
+          a = lbl.array
+          if !a.size
             next
           end
           
-          nfaStates = moveMap[lbl]
-          if !nfaStates
-            nfaStates = Set.new
-            moveMap[lbl] = nfaStates
-          end
-          nfaStates.add(dest)
+          primeId = a[0]
+          
+          next if primeId >= EPSILON-1
+          
+          lbl.difference!(CodeSet.new(primeId+1, EPSILON))
         end
       end
       
-      moveMap.each_pair do |charRange,nfaStates|
-        # May be better to test if already in set before calc closure; or simply has closure
-        epsClosure(nfaStates)
-        dfaDestState, isNew = createDFAState(stateSetToIdArray(nfaStates))
-        if isNew
-          unmarked.push(dfaDestState)
-        end
-        dfaState.addEdge(charRange, dfaDestState)
-      end
+      !db || dfa.generatePDF("minimal_dfa")
       
+      dfa
     end
     
-    if normalize
-      !db || @dfaStart.generatePDF("prior_normalize")
-      
-      !db || pr("Normalizing states for:\n\n%s\n",State.dumpNFA(@dfaStart))
-      State.normalizeStates(@dfaStart)
-      !db || pr("After normalizing:\n\n%s\n",State.dumpNFA(@dfaStart))
-      !db || @dfaStart.generatePDF("post_normalize")
-    end
       
-    @dfaStart
-  end
-  
-  private
-  
-  # Adds a DFA state for a set of NFA states, if one doesn't already exist
-  # for the set
-  # @param nfaStateList a sorted array of NFA state ids 
-  # @return a pair [DFA State, 
-  #                 created flag (boolean): true if this did not already exist]
-  #
-  def createDFAState(nfaStateList)
-    
-    lst = nfaStateList
     
-    newState = @nfaStateMap[lst]
-    isNewState = !newState
-    if isNewState
-      newState = State.new(@nextId)
+    # Constructs a builder object
+    #
+    def initialize(nfaStartState)
+      @nextId = 0
+      @nfaStart = nfaStartState
       
-      # Determine if any of the NFA states were final states
-      newState.finalState = nfaStateList.any?{|id| @nfaStateMap[id].finalState?}
+      # Build a map of nfa state ids => nfa states
+      @nfaStateMap = {}
+      nfas, _, _ = @nfaStart.reachableStates
+      nfas.each {|s| @nfaStateMap[s.id] = s}
       
-      if false
-        # Set label of DFA state to show which NFA states produced it
-        # (useful for debugging)
-        newState.label = lst.map {|x| x.to_s}.join(' ')
-      end
+      # Initialize an array of nfa state lists, indexed by dfa state id
+      @nfaStateLists = []
       
-      @nextId += 1
-      @nfaStateMap[lst] = newState   
-      @nfaStateLists.push(lst)
-        
+      # Map of existing DFA states; key is array of NFA state ids
+      @dfaStateMap = {}
     end
-    return [newState,isNewState]
-  end
   
-  def stateSetToIdArray(s)
-    s.to_a.map {|x| x.id}.sort
-  end
+    # Perform the build algorithm
+    #
+    # @param partition if true, partitions the edge labels into disjoint code sets
+    # @param normalize if true, normalizes the states afterward
+    #
+    def build(partition = true, normalize = true)
+      db = false
+      
+      !partition || partitionEdges(@nfaStart)
+      
+      iset = Set.new
+      iset.add(@nfaStart)
+      epsClosure(iset)
+      
+      @dfaStart,_ = createDFAState(stateSetToIdArray(iset))
+      
+      markedStates = Set.new
+      
+      unmarked = [@dfaStart]
   
-  # Calculate the epsilon closure of a set of NFA states
-  # @return a set of states
-  #
-  def epsClosure(stateSet)
-    stk = stateSet.to_a
-    while !stk.empty?
-      s = stk.pop
-      s.edges.each do |lbl,dest|
-        if lbl.contains? EPSILON
-          if stateSet.add?(dest)
-            stk.push(dest)
+      until unmarked.empty? 
+        dfaState  = unmarked.pop
+        
+        nfaIds = @nfaStateLists[dfaState.id]
+        
+        # map of CodeSet => set of NFA states
+        moveMap = {}
+        
+        nfaIds.each do |nfaId| 
+          nfaState = @nfaStateMap[nfaId]
+          nfaState.edges.each do |lbl,dest| 
+            if lbl.array[0] == EPSILON
+              next
+            end
+            
+            nfaStates = moveMap[lbl]
+            if !nfaStates
+              nfaStates = Set.new
+              moveMap[lbl] = nfaStates
+            end
+            nfaStates.add(dest)
           end
         end
+        
+        moveMap.each_pair do |charRange,nfaStates|
+          # May be better to test if already in set before calc closure; or simply has closure
+          epsClosure(nfaStates)
+          dfaDestState, isNew = createDFAState(stateSetToIdArray(nfaStates))
+          if isNew
+            unmarked.push(dfaDestState)
+          end
+          dfaState.addEdge(charRange, dfaDestState)
+        end
+        
       end
-    end  
-    stateSet
-  end
-  
-  # Modify edges so each is labelled with a disjoint subset
-  # of characters.  See the notes at the start of this class,
-  # as well as RangePartition.rb.
-  #
-  def partitionEdges(startState)
-   
-    db = false
-    
-    par = RangePartition.new
+      
+      if normalize
+        !db || @dfaStart.generatePDF("prior_normalize")
+        
+        !db || pr("Normalizing states for:\n\n%s\n",State.dumpNFA(@dfaStart))
+        State.normalizeStates(@dfaStart)
+        !db || pr("After normalizing:\n\n%s\n",State.dumpNFA(@dfaStart))
+        !db || @dfaStart.generatePDF("post_normalize")
+      end
+        
+      @dfaStart
+    end
     
-    stateSet, _, _ = startState.reachableStates
+    private
     
-    stateSet.each do |s|
-      s.edges.each {|lbl,dest| par.addSet(lbl) }
+    # Adds a DFA state for a set of NFA states, if one doesn't already exist
+    # for the set
+    # @param nfaStateList a sorted array of NFA state ids 
+    # @return a pair [DFA State, 
+    #                 created flag (boolean): true if this did not already exist]
+    #
+    def createDFAState(nfaStateList)
+      
+      lst = nfaStateList
+      
+      newState = @nfaStateMap[lst]
+      isNewState = !newState
+      if isNewState
+        newState = State.new(@nextId)
+        
+        # Determine if any of the NFA states were final states
+        newState.finalState = nfaStateList.any?{|id| @nfaStateMap[id].finalState?}
+        
+        if false
+          # Set label of DFA state to show which NFA states produced it
+          # (useful for debugging)
+          newState.label = lst.map {|x| x.to_s}.join(' ')
+        end
+        
+        @nextId += 1
+        @nfaStateMap[lst] = newState   
+        @nfaStateLists.push(lst)
+          
+      end
+      return [newState,isNewState]
     end
     
-    par.prepare
+    def stateSetToIdArray(s)
+      s.to_a.map {|x| x.id}.sort
+    end
     
-    stateSet.each do |s|
-      newEdges = []
-      s.edges.each do |lbl, dest|
-        !db||pr(" old edge: %s   => %s\n",d(lbl),d(dest.name))
-        newLbls = par.apply(lbl)
-        newLbls.each {|x| newEdges.push([x, dest]) }
+    # Calculate the epsilon closure of a set of NFA states
+    # @return a set of states
+    #
+    def epsClosure(stateSet)
+      stk = stateSet.to_a
+      while !stk.empty?
+        s = stk.pop
+        s.edges.each do |lbl,dest|
+          if lbl.contains? EPSILON
+            if stateSet.add?(dest)
+              stk.push(dest)
+            end
+          end
+        end
       end  
-      s.clearEdges()
+      stateSet
+    end
+    
+    # Modify edges so each is labelled with a disjoint subset
+    # of characters.  See the notes at the start of this class,
+    # as well as RangePartition.rb.
+    #
+    def partitionEdges(startState)
+     
+      db = false
+      
+      par = RangePartition.new
+      
+      stateSet, _, _ = startState.reachableStates
+      
+      stateSet.each do |s|
+        s.edges.each {|lbl,dest| par.addSet(lbl) }
+      end
+      
+      par.prepare
       
-      newEdges.each do |lbl,dest| 
-        !db||pr(" new edge: %s   => %s\n",d(lbl),d(dest.name))
-        s.addEdge(lbl,dest)
+      stateSet.each do |s|
+        newEdges = []
+        s.edges.each do |lbl, dest|
+          !db||pr(" old edge: %s   => %s\n",d(lbl),d(dest.name))
+          newLbls = par.apply(lbl)
+          newLbls.each {|x| newEdges.push([x, dest]) }
+        end  
+        s.clearEdges()
+        
+        newEdges.each do |lbl,dest| 
+          !db||pr(" new edge: %s   => %s\n",d(lbl),d(dest.name))
+          s.addEdge(lbl,dest)
+        end
+        !db||pr("\n")
       end
-      !db||pr("\n")
+      
     end
+  
     
   end
 
-  
-end
-
-
+end  # module ToknInternal