tokn 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d9e7bb7fabdbd657fefe13b23eeddfff4493dd76
-  data.tar.gz: e1c7c681b4a4928002de7472ab7c8bc0132c2b4e
+  metadata.gz: 75d7206c10817a05dbbd9e5ff36b25f20ef5ad18
+  data.tar.gz: d1d84da2be85c05b567b476b53471ac2a7c6a04f
 SHA512:
-  metadata.gz: 3d217315f1b9559b58b4bef86c92402076f67323cd7715693425aee26fceb8c5ca2dfa021b4678303103cb678c9fe47f5978d2cbc2dc07e6060c0678e11883c2
-  data.tar.gz: 15d5d9577c4402a593c19fb522738b996a4cdb7b01ee4c7d576a64f07aa74f31c40917cdefdd48b35c28608abf6425bfcf59cf59e243774cc17e01259cb2c8bd
+  metadata.gz: 47a078062419175ef5fd1d66a46bae95b423c1b69cd8d27a28875589b13abdcdd5474c81e70a9b09f11a27b302f9f9038302fd1ad9121bf81d7db42cf20cea96
+  data.tar.gz: 2bd489cb22b1b68d0a365fd70c337e7cc3bfd1872d501a88e735fe23fe3780a77415262be265263cd906eb7cf32afea7f5e1c8cb849c6feef44567ef80b4717b

data/README.txt

@@ -66,7 +66,9 @@ definitions shown earlier.
 
 	require "Tokenizer"
 	
-	dfa = dfa_from_script(readTextFile("tokendefs.txt"))
+	include Tokn   # Avoids having to prefix things with 'Tokn::'
+	
+	dfa = DFA.from_script(readTextFile("tokendefs.txt"))
 	
 	t = Tokenizer.new(dfa, readTextFile("source.txt"))
 	
@@ -92,7 +94,7 @@ The module has two utility scripts: tokncompile, and toknprocess.  These can be
 found in the bin/ directory.
 
 The tokncompile script reads a token definition script from standard input, and
-compiles it to a DFA.  For example, if you are in the tokn directory, you can 
+compiles it to a DFA.  For example, if you are in the tokn/test/data directory, you can 
 type:
   
   tokncompile < sampletokens.txt > compileddfa.txt
@@ -189,6 +191,3 @@ only reads Ruby characters from the input, which I believe are only 8 bits wide.
     
 Well, I can be reached as jpsember at gmail dot com.
  
- 
-
-

data/bin/tokncompile

@@ -11,6 +11,6 @@
 
 require 'tokn'
 
-puts dfa_from_script(ARGF.read).serialize()
+puts Tokn::DFA.from_script(ARGF.read).serialize()
 
 

data/bin/toknprocess

@@ -11,14 +11,20 @@
 #
 
 require 'tokn'
+include Tokn
 
-if ARGV.size != 2
-  puts "Usage: toknprocess <dfa file> <source file>"
+if ARGV.size < 2 || ARGV.size > 3
+  puts "Usage: toknprocess <dfa file> <source file> [<skip token name>]"
   abort
 end
 
-dfa = dfa_from_file(ARGV[0])
-tk = Tokenizer.new(dfa, readTextFile(ARGV[1]))
+dfa = DFA.from_file(ARGV[0])
+skipName = nil
+if ARGV.size >= 3
+	skipName = ARGV[2]
+end
+
+tk = Tokenizer.new(dfa, readTextFile(ARGV[1]), skipName)
 
 while tk.hasNext()
   t = tk.read

data/lib/tokn/code_set.rb

@@ -1,392 +1,387 @@
 require_relative 'tools'
 
-req('tokn_const')
-
-
-# A CodeSet is an ordered set of character or token codes that
-# are used as labels on DFA edges.
-#
-# In addition to unicode character codes 0...0x10ffff, they
-# also represent epsilon transitions (-1), or token identifiers ( < -1).
-#
-# Each CodeSet is represented as an array with 2n elements;
-# each pair represents a closed lower and open upper range of values.
-# 
-# Thus a value x is within the set [a1,a2,b1,b2,..]
-# iff (a1 <= x < a2) or (b1 <= x < b2) or ...
-#
-class CodeSet
-  
-  include Tokn
- 
-  # Construct a copy of this set
-  #
-  def makeCopy
-    c = CodeSet.new
-    c.setTo(self)
-    c
-  end
+module ToknInternal
   
-  # Initialize set; optionally add an initial contiguous range
-  #
-  def initialize(lower = nil, upper = nil)
-    @elem = []
-    if lower
-      add(lower,upper)
-    end
-  end
-  
-  # Replace this set with a copy of another
-  #
-  def setTo(otherSet)
-    @elem.replace(otherSet.array)  
-  end
-  
-  # Get the array containing the code set range pairs
-  #
-  def array
-    return @elem
-  end
-  
-  # Replace this set's array
-  # @param a array to point to (does not make a copy of it)
-  #
-  def setArray(a)
-    @elem = a
-  end
-  
-
-  def hash
-    return @elem.hash
-  end
-
-  # Determine if this set is equivalent to another
+  # A CodeSet is an ordered set of character or token codes that
+  # are used as labels on DFA edges.
   #
-  def eql?(other)
-    @elem == other.array
-  end
-
-
-  # Add a contiguous range of values to the set
-  # @param lower min value in range
-  # @param upper one plus max value in range
+  # In addition to unicode character codes 0...0x10ffff, they
+  # also represent epsilon transitions (-1), or token identifiers ( < -1).
   #
-  def add(lower, upper = nil)
-    if not upper
-      upper = lower + 1
-    end
-    
-    if lower >= upper
-      raise RangeError
-    end
-    
-    newSet = [] 
-    i = 0
-    while i < @elem.size and @elem[i] < lower
-      newSet.push(@elem[i])
-      i += 1
-    end 
-    
-    if (i & 1) == 0
-      newSet.push(lower)
-    end
-    
-    while i < @elem.size and @elem[i] <= upper
-      i += 1
-    end
-    
-    if (i & 1) == 0
-      newSet.push(upper)
-    end
-    
-    while i < @elem.size 
-      newSet.push(@elem[i])
-      i += 1
-    end
-    
-    @elem = newSet
-    
-  end
-
-
-
-
-
-
-  # Remove a contiguous range of values from the set
-  # @param lower min value in range
-  # @param upper one plus max value in range
+  # Each CodeSet is represented as an array with 2n elements;
+  # each pair represents a closed lower and open upper range of values.
+  # 
+  # Thus a value x is within the set [a1,a2,b1,b2,..]
+  # iff (a1 <= x < a2) or (b1 <= x < b2) or ...
   #
-  def remove(lower, upper = nil)
-    if not upper
-      upper = lower + 1
+  class CodeSet
+    
+    # Construct a copy of this set
+    #
+    def makeCopy
+      c = CodeSet.new
+      c.setTo(self)
+      c
     end
     
-    if lower >= upper
-      raise RangeError
+    # Initialize set; optionally add an initial contiguous range
+    #
+    def initialize(lower = nil, upper = nil)
+      @elem = []
+      if lower
+        add(lower,upper)
+      end
     end
     
-    newSet = [] 
-    i = 0
-    while i < @elem.size and @elem[i] < lower
-      newSet.push(@elem[i])
-      i += 1
-    end 
-    
-    if (i & 1) == 1
-      newSet.push(lower)
+    # Replace this set with a copy of another
+    #
+    def setTo(otherSet)
+      @elem.replace(otherSet.array)  
     end
     
-    while i < @elem.size and @elem[i] <= upper
-      i += 1
+    # Get the array containing the code set range pairs
+    #
+    def array
+      @elem
     end
     
-    if (i & 1) == 1
-      newSet.push(upper)
+    # Replace this set's array
+    # @param a array to point to (does not make a copy of it)
+    #
+    def setArray(a)
+      @elem = a
     end
-    
-    while i < @elem.size 
-      newSet.push(@elem[i])
-      i += 1
+  
+    # Get hash code; just uses hash code of the contained array
+    def hash
+      @elem.hash
     end
-    
-    setArray(newSet)
-    
-  end
-
-  # Replace this set with itself minus another
-  #
-  def difference!(s)
-    setTo(difference(s))
-  end
   
-  # Calculate difference of this set minus another
-  def difference(s)
-    combineWith(s, 'd')
-  end
-
-  # Calculate the intersection of this set and another
-  def intersect(s)
-    combineWith(s, 'i')
-  end
-
-
-
-  # Set this set equal to its intersection with another
-  def intersect!(s)
-    setTo(intersect(s))
-  end
+    # Determine if this set is equivalent to another, by
+    # comparing the contained arrays
+    #
+    def eql?(other)
+      @elem == other.array
+    end
   
-  # Add every value from another CodeSet to this one
-  def addSet(s)
-    sa = s.array
-    
-    (0 ... sa.length).step(2) {
-      |i| add(sa[i],sa[i+1])
-    }
-  end
   
-  # Determine if this set contains a particular value
-  def contains?(val)
-    ret = false
-    i = 0
-    while i < @elem.size
-      if val < @elem[i]
-        break
+    # Add a contiguous range of values to the set
+    # @param lower min value in range
+    # @param upper one plus max value in range
+    #
+    def add(lower, upper = nil)
+      if not upper
+        upper = lower + 1
+      end
+      
+      if lower >= upper
+        raise RangeError
+      end
+      
+      newSet = [] 
+      i = 0
+      while i < @elem.size and @elem[i] < lower
+        newSet.push(@elem[i])
+        i += 1
       end 
-      if val < @elem[i+1]
-        ret = true
-        break
+      
+      if (i & 1) == 0
+        newSet.push(lower)
       end
-      i += 2
-    end  
-    
-    ret
-    
-  end
-  
-  # Get string representation of set, treating them (where
-  # possible) as printable ASCII characters
-  #
-  def to_s
-    s = ''
-    i = 0
-    while i < @elem.size
-      if s.size
-        s += ' '
+      
+      while i < @elem.size and @elem[i] <= upper
+        i += 1
       end
       
-      lower = @elem[i]
-      upper = @elem[i+1]
-      s += dbStr(lower)
-      if upper != 1+lower
-        s += '..' + dbStr(upper-1)
+      if (i & 1) == 0
+        newSet.push(upper)
       end
-      i += 2
+      
+      while i < @elem.size 
+        newSet.push(@elem[i])
+        i += 1
+      end
+      
+      @elem = newSet
+      
     end
-    return s
-  end
   
-  def inspect
-    to_s
-  end
   
-  # Get string representation of set, treating them
-  # as integers
-  #
-  def to_s_alt
-    s = ''
-    i = 0
-    while i < @elem.size
-      if s.length > 0
-        s += ' '
+    # Remove a contiguous range of values from the set
+    # @param lower min value in range
+    # @param upper one plus max value in range
+    #
+    def remove(lower, upper = nil)
+      if not upper
+        upper = lower + 1
       end
-      low = @elem[i]
-      upr = @elem[i+1]
-      s += low.to_s
-      if upr > low+1
-        s += '..'
-        s += (upr-1).to_s
+      
+      if lower >= upper
+        raise RangeError
       end
-      i += 2
+      
+      newSet = [] 
+      i = 0
+      while i < @elem.size and @elem[i] < lower
+        newSet.push(@elem[i])
+        i += 1
+      end 
+      
+      if (i & 1) == 1
+        newSet.push(lower)
+      end
+      
+      while i < @elem.size and @elem[i] <= upper
+        i += 1
+      end
+      
+      if (i & 1) == 1
+        newSet.push(upper)
+      end
+      
+      while i < @elem.size 
+        newSet.push(@elem[i])
+        i += 1
+      end
+      
+      setArray(newSet)
+      
     end
-    return s
-  end
-  
   
-  # Negate the inclusion of a contiguous range of values
-  #
-  # @param lower min value in range
-  # @param upper one plus max value in range
-  #
-  def negate(lower = 0, upper =  CODEMAX)
-    s2 = CodeSet.new(lower,upper)
-    if lower >= upper
-      raise RangeError
+    # Replace this set with itself minus another
+    #
+    def difference!(s)
+      setTo(difference(s))
     end
     
-    newSet = [] 
-    i = 0
-    while i < @elem.size and @elem[i] <= lower
-      newSet.push(@elem[i])
-      i += 1
-    end 
-    
-    if i > 0 and newSet[i-1] == lower
-      newSet.pop
-    else
-      newSet.push(lower)
+    # Calculate difference of this set minus another
+    def difference(s)
+      combineWith(s, 'd')
     end
-    
-    while i < @elem.size and @elem[i] <= upper
-      newSet.push(@elem[i])
-      i += 1
-    end 
-    
-    
-    if newSet.length > 0 and newSet.last == upper
-      newSet.pop
-    else
-      newSet.push(upper)
-    end
-    
-    while i < @elem.size 
-      newSet.push(@elem[i])
-      i += 1
-    end 
-    
-    @elem = newSet
-    
-  end
   
-  # Determine how many distinct values are represented by this set
-  def cardinality
-    c = 0
-    i = 0
-    while i < @elem.length
-      c += @elem[i+1] - @elem[i]
-      i += 2
+    # Calculate the intersection of this set and another
+    def intersect(s)
+      combineWith(s, 'i')
     end
-    c
-  end
   
-  # Determine if this set is empty
-  #
-  def empty?
-    @elem.empty?
-  end
   
-  private
-  
-  # Get a debug description of a value within a CodeSet, suitable
-  # for including within a .dot label
-  #
-  def dbStr(charCode)
+    # Set this set equal to its intersection with another
+    def intersect!(s)
+      setTo(intersect(s))
+    end
     
-    # Unless it corresponds to a non-confusing printable ASCII value,
-    # just print its decimal equivalent
+    # Add every value from another CodeSet to this one
+    def addSet(s)
+      sa = s.array
+      
+      (0 ... sa.length).step(2) {
+        |i| add(sa[i],sa[i+1])
+      }
+    end
+    
+    # Determine if this set contains a particular value
+    def contains?(val)
+      ret = false
+      i = 0
+      while i < @elem.size
+        if val < @elem[i]
+          break
+        end 
+        if val < @elem[i+1]
+          ret = true
+          break
+        end
+        i += 2
+      end  
+      
+      ret
+      
+    end
     
-    s = charCode.to_s
+    # Get string representation of set, treating them (where
+    # possible) as printable ASCII characters
+    #
+    def to_s
+      s = ''
+      i = 0
+      while i < @elem.size
+        if s.size
+          s += ' '
+        end
+        
+        lower = @elem[i]
+        upper = @elem[i+1]
+        s += dbStr(lower)
+        if upper != 1+lower
+          s += '..' + dbStr(upper-1)
+        end
+        i += 2
+      end
+      return s
+    end
     
-    if charCode == EPSILON
-      s = "(e)"
-    elsif (charCode > 32 && charCode < 0x7f && !"'\"\\[]{}()".index(charCode.chr))
-      s = charCode.chr
-    end  
-    return s
-  end
-
-  # Combine this range (a) with another (b) according to particular operation 
-  # > s  other range (b)
-  # > oper   'i': intersection, a^b
-  #          'd': difference, a-b
-  #          'n': negation, (a & !b) | (!a & b)
-  # 
-  def combineWith(s, oper)
-    sa = array
-    sb = s.array
+    # Calls to_s
+    def inspect
+      to_s
+    end
     
-    i = 0
-    j = 0
-    c = []
+    # Get string representation of set, treating them
+    # as integers
+    #
+    def to_s_alt
+      s = ''
+      i = 0
+      while i < @elem.size
+        if s.length > 0
+          s += ' '
+        end
+        low = @elem[i]
+        upr = @elem[i+1]
+        s += low.to_s
+        if upr > low+1
+          s += '..'
+          s += (upr-1).to_s
+        end
+        i += 2
+      end
+      return s
+    end
     
-    wasInside = false
     
-    while i < sa.length || j < sb.length
+    # Negate the inclusion of a contiguous range of values
+    #
+    # @param lower min value in range
+    # @param upper one plus max value in range
+    #
+    def negate(lower = 0, upper =  CODEMAX)
+      s2 = CodeSet.new(lower,upper)
+      if lower >= upper
+        raise RangeError
+      end
       
-      if i == sa.length 
-        v = sb[j]
-      elsif j == sb.length
-        v = sa[i]
+      newSet = [] 
+      i = 0
+      while i < @elem.size and @elem[i] <= lower
+        newSet.push(@elem[i])
+        i += 1
+      end 
+      
+      if i > 0 and newSet[i-1] == lower
+        newSet.pop
       else
-        v = [sa[i],sb[j]].min
+        newSet.push(lower)
       end
-
-      if i < sa.length && v == sa[i]
+      
+      while i < @elem.size and @elem[i] <= upper
+        newSet.push(@elem[i])
         i += 1
-      end      
-      if j < sb.length && v == sb[j]
-        j += 1
-      end      
-      
-      case oper
-      when 'i'
-        inside = ((i & 1) == 1) && ((j & 1) == 1)
-      when 'd'
-        inside = ((i & 1) == 1) && ((j & 1) == 0)
+      end 
+      
+      
+      if newSet.length > 0 and newSet.last == upper
+        newSet.pop
       else
-        raise Exception, "illegal"
+        newSet.push(upper)
+      end
+      
+      while i < @elem.size 
+        newSet.push(@elem[i])
+        i += 1
+      end 
+      
+      @elem = newSet
+      
+    end
+    
+    # Determine how many distinct values are represented by this set
+    def cardinality
+      c = 0
+      i = 0
+      while i < @elem.length
+        c += @elem[i+1] - @elem[i]
+        i += 2
       end
+      c
+    end
+    
+    # Determine if this set is empty
+    #
+    def empty?
+      @elem.empty?
+    end
+    
+    private
+    
+    # Get a debug description of a value within a CodeSet, suitable
+    # for including within a .dot label
+    #
+    def dbStr(charCode)
       
-      if inside != wasInside
-        c.push v
-        wasInside = inside
+      # Unless it corresponds to a non-confusing printable ASCII value,
+      # just print its decimal equivalent
+      
+      s = charCode.to_s
+      
+      if charCode == EPSILON
+        s = "(e)"
+      elsif (charCode > 32 && charCode < 0x7f && !"'\"\\[]{}()".index(charCode.chr))
+        s = charCode.chr
+      end  
+      return s
+    end
+  
+    # Combine this range (a) with another (b) according to particular operation 
+    # > s  other range (b)
+    # > oper   'i': intersection, a^b
+    #          'd': difference, a-b
+    #          'n': negation, (a & !b) | (!a & b)
+    # 
+    def combineWith(s, oper)
+      sa = array
+      sb = s.array
+      
+      i = 0
+      j = 0
+      c = []
+      
+      wasInside = false
+      
+      while i < sa.length || j < sb.length
+        
+        if i == sa.length 
+          v = sb[j]
+        elsif j == sb.length
+          v = sa[i]
+        else
+          v = [sa[i],sb[j]].min
+        end
+  
+        if i < sa.length && v == sa[i]
+          i += 1
+        end      
+        if j < sb.length && v == sb[j]
+          j += 1
+        end      
+        
+        case oper
+        when 'i'
+          inside = ((i & 1) == 1) && ((j & 1) == 1)
+        when 'd'
+          inside = ((i & 1) == 1) && ((j & 1) == 0)
+        else
+          raise Exception, "illegal"
+        end
+        
+        if inside != wasInside
+          c.push v
+          wasInside = inside
+        end
       end
+      ret = CodeSet.new()
+      ret.setArray(c)
+      ret
     end
-    ret = CodeSet.new()
-    ret.setArray(c)
-    ret
+  
   end
 
-end
-
+end