tokn 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84933fe2966d19908c447c84cbccb9179c4e351d
-  data.tar.gz: d1a6fae299cdd6c9b57961bfde7c879e1a786a48
+  metadata.gz: 84244706589683dd8fa77c96355a4494d6585561
+  data.tar.gz: cf5dcd362908eaad20371f9f62193000fc986d8b
 SHA512:
-  metadata.gz: 022689701816eb3fb37690579b275194cbafc53d31591b7fb87d93a6cacafc50318dd964d28697ed1181ceb288d700214191cc41d6362cf4221e110e7c885531
-  data.tar.gz: 96ee32c79d3e12ba8b01cd27fdfb9ff0bfc6ff518972358dc2fbac04f4c8842fccd6f8722c994daba299327718915074f853cc3c9125c0e22dffe3ef19fd189d
+  metadata.gz: c2a6659f5c4819088ff95dda3a48b42b2a7deb7dcd46f01149b45121bff298778850e42597e63a3cd8ec5bcc94f9aba41c90b914c2b5d88f6cf13ad9d64dd4cf
+  data.tar.gz: 059e6c6601c8e35aa7f7e465fd408dc227a23e899458962592a12bdd4bcc9004bb802bf0ca4e580e8502c6a2ba2ebe4ea8e23705903198efd431e30833783cb6

data/CHANGELOG.txt

@@ -16,4 +16,5 @@
 	* Version 0.1.0
 	* Cleaned up test directory
 	
-	
+2013-04-15
+  * No longer maintaining CHANGELOG.txt; see git commit comments instead.	

data/lib/tokn/dfa.rb

@@ -20,7 +20,7 @@ module Tokn
     def self.from_script(script, persistPath = nil)
       
       if persistPath and File.exist?(persistPath)
-        return extractDFA(readTextFile(persistPath))
+        return extractDFA(read_text_file(persistPath))
       end
       
       req('token_defn_parser')
@@ -29,7 +29,7 @@ module Tokn
       dfa = td.dfa
       
       if persistPath
-        writeTextFile(persistPath, dfa.serialize())
+        write_text_file(persistPath, dfa.serialize())
       end
     
       dfa  
@@ -39,14 +39,14 @@ module Tokn
     # the file at scriptPath.
     #
     def self.from_script_file(scriptPath, persistPath = nil)
-      self.from_script(readTextFile(scriptPath), persistPath)  
+      self.from_script(read_text_file(scriptPath), persistPath)  
     end
     
     # Compile a Tokenizer DFA from a text file (that contains a
     # JSON string)
     #
     def self.from_file(path)
-      from_json(readTextFile(path))
+      from_json(read_text_file(path))
     end
     
     # Compile a Tokenizer DFA from a JSON string

data/lib/tokn/tokn_const.rb

@@ -32,3 +32,18 @@ module ToknInternal
     EPSILON-1-edgeLabel
   end
 end
+
+# Convert a .dot file (string) to a PDF file "__mygraph__nnn.pdf" 
+# 
+# It does this by making a system call to the 'dot' utility.
+#
+def dotToPDF(dotFile, name = "", test_dir = nil)
+  gr = dotFile
+  
+  raise ArgumentError if !test_dir
+  
+  dotPath = File.join(test_dir,".__mygraph__.dot")
+  write_text_file(dotPath,gr)
+  destName = File.join(test_dir,"__mygraph__"+name+".pdf")
+  system("dot -Tpdf "+dotPath+" -o "+destName)
+end

data/lib/tokn/tools.rb

@@ -1,13 +1,31 @@
 require 'set'
 require 'fileutils'
 
+###############################################################
+#
 # Various utility and debug convenience functions.
 #
+###############################################################
 
-# Perform 'require_relative' on a set of files
+# Exception class for objects in illegal states
 #
-# fileListStr : space-delimited file/path items, without .rb extensions
-# subdir : optional path to files relative to tools.rb
+class IllegalStateException < Exception
+end
+
+# A string containing a single zero, with ASCII 8-bit encoding (i.e., plain old bytes)
+ZERO_CHAR = "\0".force_encoding("ASCII-8BIT")
+
+# Construct a string of zeros
+# @param count number of zeros
+#
+def zero_bytes(count)
+  ZERO_CHAR * count
+end
+
+# Convenience method to perform 'require_relative' on a set of files
+#
+# @param fileListStr  space-delimited file/path items, without .rb extensions
+# @param subdir  optional path to files relative to this file
 #
 def req(fileListStr,subdir = nil)
   fileListStr.split(' ').each do |x|
@@ -21,67 +39,114 @@ end
 
 # Shorthand for printf(...)
 #
-def pr(*args)
-  printf(*args)
+alias :pr :printf
+
+# Convert an object to a human-readable string,
+# or <nil>; should be considered a debug-only feature
+# 
+def d(arg)
+  arg.nil? ? "<nil>" : arg.inspect
 end
 
+# Convert an object to a human-readable string,
+# by calling a type-appropriate function: da, dh, or just d.
+# @param arg object
+# @param indent optional indentation for pretty printing; if result
+#          spans multiple lines, each line should be indented by this amount
+#
+def d2(arg, indent = 0)
+  return da(arg, indent) if arg.is_a? Array
+  return dh(arg, indent) if arg.is_a? Hash
+  return df(arg) if arg.class == FalseClass || arg.class == TrueClass
+  return d(arg)  
+end
 
-# Convert an object to a human-readable string;
-# should be considered a debug-only feature
+# Convert an object to a human-readable string, prefixed with its type
 #
-def d(arg)
-  arg.nil? ? "<nil>" : arg.inspect
+def dt(arg)
+  if arg.nil?
+    return "<nil>"
+  end
+  s = arg.class.to_s
+  s << ':'
+  s << arg.inspect
+  s
+end
+
+# Append a particular number of spaces to a string
+def add_sp(s, indent = 0)
+  s << ' ' * indent
+end
+
+# Pretty-print an array,
+# one element to a line
+# @param indent indentation of each line, in spaces
+def da(array, indent = 0)
+  return d(array) if !array
+  s = 'Array ['
+  indent += 2
+  array.each do |x| 
+    s << "\n"
+    add_sp(s,indent)
+    s2 = d2(x, indent + 2)
+    s << s2
+  end
+  s << " ]"
+  s
+end
+
+# Pretty-print a hash,
+# one element to a line
+# @param indent indentation of each line, in spaces
+def dh(hash, indent = 0)
+  return d(hash) if !hash
+  s = 'Hash {'
+  indent += 2
+  hash.each_pair do |key,val| 
+    s2 = d(key)
+    s3 = d2(val, indent + 4)
+    s << "\n " 
+    add_sp(s,indent)
+    s << s2.chomp << " => " << s3.chomp
+  end
+  s << " }"
+  s
+end
+
+# Generate debug description of a boolean value
+# @param flag value to interpret as a boolean; prints 'T' iff not nil
+# @param label optional label 
+def df(flag, label=nil)
+  s = ''
+  if label
+    s << label << ':'
+  end
+  s << (flag ? "T" : "F")
+  s << ' '
+  s
 end
 
 # Assert that a value is true.  Should be considered a 
 # very temporary, debug-only option; it is slow and
 # generates a warning that it is being called.
-#
-def myAssert(cond, *msg)
-  oneTimeAlert("warning",0,"Checking assertion")
+# @param cond condition
+# @param msg generates additional message using printf(), if these arguments exist
+def assert!(cond, *msg)
+  one_time_alert("warning",0,"Checking assertion")
   if not cond
     str = (msg.size == 0) ? "assertion error" : sprintf(*msg)
     raise Exception, str
   end
 end
 
-
-## Set test directory.  If nil, sets to home directory + "__test__"
-##
-#def setTestDir(d = nil)
-#  $testDir = d || File.join(Dir.home,"__test__")
-#end
- 
-## Get a path within the test directory;
-## create test directory if it doesn't exist.
-##
-## relPath : if nil, returns the test directory; else 
-##   returns the test directory joined to this one
-##
-#def withinTestDir(relPath = nil)
-#  if !$testDir
-#    raise IllegalStateException, "No test directory has been defined"
-#  end
-#  if !File.directory?($testDir)  
-#    Dir::mkdir($testDir)
-#  end
-#  relPath ? File.join($testDir,relPath) : $testDir
-#end
-
-# Convert a .dot file (string) to a PDF file "__mygraph__nnn.pdf" 
-# in the test directory.
-# 
-# It does this by making a system call to the 'dot' utility.
+# Abort with message about unimplemented code
 #
-def dotToPDF(dotFile, name = "", test_dir = nil)
-  gr = dotFile
-  
-  raise ArgumentError if !test_dir
-  
-  dotPath = File.join(test_dir,".__mygraph__.dot")
-  writeTextFile(dotPath,gr)
-  destName = File.join(test_dir,"__mygraph__"+name+".pdf")
-  system("dot -Tpdf "+dotPath+" -o "+destName)
+def unimp!(msg = nil)
+  msg2 = "Unimplemented code"
+  if msg
+    msg2 << ": " << msg
+  end
+  raise Exception, msg2
 end
 
 # Extensions to the Enumerable module
@@ -106,13 +171,12 @@ module Enumerable
   end
 end
 
-
 # Get a nice, concise description of the file and line
 # of some caller within the stack.
 # 
-#  nSkip : the number of items deep in the call stack to look
+# @param nSkip the number of items deep in the call stack to look
 #
-def getCallerLocation(nSkip = 2) 
+def get_caller_location(nSkip = 2) 
   
   filename = nil
   linenumber = nil
@@ -130,11 +194,11 @@ def getCallerLocation(nSkip = 2)
       if pth.size
         filename = pth[-1]
       end
-      linenumber = fi[i+1,j-i-1].to_i
+      linenumber = fi[i+1,j-i-1]  
     end
   end
   if filename && linenumber
-    loc = filename + " ("+linenumber.to_s+")"
+    loc = filename + " ("+linenumber+")"
   else 
     loc = "(UNKNOWN LOCATION)"
   end
@@ -149,12 +213,12 @@ $AlertStrings = Set.new
 # Print a message if it hasn't yet been printed,
 # which includes the caller's location
 #
-# > typeString : e.g., "warning", "unimplemented"
-# > nSkip : the number of levels deep that the caller is in the stack
-# > args : if present, calls sprintf(...) with these to append to the message
+# @param typeString  e.g., "warning", "unimplemented"
+# @param nSkip    the number of levels deep that the caller is in the stack
+# @param args    if present, calls sprintf(...) with these to append to the message
 #
-def oneTimeAlert(typeString, nSkip, *args) 
-  loc = getCallerLocation(nSkip + 2)
+def one_time_alert(typeString, nSkip, *args) 
+  loc = get_caller_location(nSkip + 2)
   s = "*** "+typeString+" " + loc
   if args && args.size
     s2 = sprintf(args[0], *args[1..-1])
@@ -169,33 +233,160 @@ def oneTimeAlert(typeString, nSkip, *args)
 end
      
 # Print a 'warning' alert, one time only 
-#   
+# @param args if present, calls printf() with these
 def warn(*args) 
-  oneTimeAlert("warning",0, *args)
+  one_time_alert("warning",0, *args)
+end
+
+# Convenience method for setting 'db' true within methods,
+# and to print a one-time warning if so.
+# @param val value to set db to; it is convenient to disable
+#    debug printing quickly by adding a zero, e.g., 'warndb 0'
+#
+def warndb(val = true)
+  if !val || val == 0
+    return false
+  end
+  one_time_alert("warning",1,"Debug printing enabled")
+  true
 end
 
 # Print an 'unimplemented' alert, one time only 
-#   
+# @param args if present, calls printf() with these
 def unimp(*args)
-  oneTimeAlert("unimplemented", 0, *args)
+  one_time_alert("unimplemented", 0, *args)
 end
 
 # Write a string to a text file
 #
-def writeTextFile(path, contents)
+def write_text_file(path, contents)
     File.open(path, "wb") {|f| f.write(contents) }
 end
 
 # Read a file's contents, return as a string
 #
-def readTextFile(path)
+def read_text_file(path)
   contents = nil
   File.open(path,"rb") {|f| contents = f.read }
   contents
 end
 
+# Method that takes a code block as an argument to 
+# achieve the same functionality as Java/C++'s
+#  do {
+#    ...
+#    ...  possibly with 'break' to jump to the end ...
+#  } while (false);
+#
+def block
+  yield
+end
+
+# Construct hex representation of value
+# @param value integer value
+# @param num_digits number of hex digits
+#
+def to_hex(value, num_digits=4) 
+  s = sprintf("%x", value)
+  s.rjust(num_digits,'0')
+end
+
+# Hex dump a string or byte array
+# @param byte_array_or_string
+# @param title
+# @param offset offset to first value within array
+# @param length number of values to dump
+# @param bytes_per_row 
+# @param with_text if true, displays ASCII values to right of hex dump
+#
+def hex_dump(byte_array_or_string, title=nil, offset=0, length= -1, bytes_per_row=16, with_text=true) 
+  ss = hex_dump_to_string(byte_array_or_string, title, offset, length, bytes_per_row, with_text)
+  puts ss
+end
+
+# Hex dump a string or byte array to a string; see hex_dump for parameter descriptions
+#
+def hex_dump_to_string(byte_array_or_string, title=nil, offset=0, length= -1, bytes_per_row=16, with_text=true)
+  
+  byte_array = byte_array_or_string
+  if byte_array.is_a? String
+    byte_array = byte_array.bytes.to_a
+  end
+  
+  ss = ''
+  
+  if title 
+    ss << title << ":\n"
+  end
+  
+  if length < 0 
+    length = byte_array.size - offset
+  end
+    
+  length = [length, byte_array.size - offset].min
+  
+  max_addr = offset + length - 1
+  num_digits = 4
+  while (1 << (4 * num_digits)) <= max_addr
+    num_digits += 1
+  end
+  
+  while true
+    ss << to_hex(offset, num_digits)
+    ss << ': '
+    
+    chunk = [length, bytes_per_row].min
+    bytes_per_row.times do |i|
+      if i % 4 == 0 
+        ss << '  '
+      end
+      
+      if i < chunk 
+        v = byte_array[offset + i]
+        ss << ((v != 0) ? to_hex(v,2) : '..')
+        ss << ' '
+      else
+        ss << '   '
+      end
+    end
+        
+    if with_text 
+      ss << '  |'
+      bytes_per_row.times do |i|
+        if i < chunk 
+          v = byte_array[offset + i]
+          ss << ((v >= 32 && v < 127) ? v : '_')
+        end
+      end
+      ss << '|'
+    end
+    ss << "\n"
+    
+    length -= chunk
+    offset += chunk
+    break if length <= 0
+  end
+  ss
+end
+
+$prevTime = nil
+
+# Calculate time elapsed, in seconds, from last call to this function;
+# if it's never been called, returns zero
+#
+def elapsed 
+  curr = Time.now.to_f
+  elap = 0
+  if $prevTime
+    elap = curr - $prevTime
+  end
+  $prevTime = curr
+  elap
+end
+
 # Delete a file or directory, if it exists.
 # Caution!  If directory, deletes all files and subdirectories.
+#
 def remove_file_or_dir(pth)
   if File.directory?(pth)
     FileUtils.remove_dir(pth)
@@ -204,22 +395,96 @@ def remove_file_or_dir(pth)
   end
 end  
 
-# Method that takes a code block as an argument to 
-# achieve the same functionality as Java/C++'s
-#  do {
-#    ...
-#    ...  possibly with 'break' to jump to the end ...
-#  } while (false);
+require 'stringio'
+
+$IODest = nil
+$OldStdOut = nil
+
+# Redirect standard output to an internal string
 #
-def block
-  yield
+def capture_begin
+    raise IllegalStateException if $IODest
+    $IODest = StringIO.new
+    $OldStdOut, $stdout = $stdout, $IODest
 end
 
-# Exception class for objects in illegal states
+# Restore standard output; return captured text
+# @return text that was redirected
 #
-class IllegalStateException < Exception
+def capture_end
+  raise IllegalStateException if !$IODest
+  $stdout = $OldStdOut  
+  ret = $IODest.string
+  $IODest = nil
+  ret
+end
+
+# Compare a string with disk file; abort if different.  Disk filename is derived
+# from caller function name; e.g., test_xxx produces filename _output_xxx
+#
+# @param str if not nil, string to compare; if nil, calls capture_end to get string
+#
+def match_expected_output(str = nil)
+
+  if !str
+    str = capture_end
+  end
+
+  cl_method = caller[0][/`.*'/][1..-2]
+  if (cl_method.start_with?("test_"))
+    cl_method = cl_method[5..-1]
+  end
+  path = "_output_" + cl_method + ".txt"
+
+  if !File.file?(path)
+    printf("no such file #{path} exists, writing it...\n")
+    write_text_file(path,str)
+  else
+    exp_cont = read_text_file(path)
+    if str != exp_cont
+      d1 = str
+      d2 = exp_cont
+
+      # Find location where they differ
+      lines1 = d1.split("\n")
+      lines2 = d2.split("\n")
+      j = [lines1.size, lines2.size].max
+
+      s = "???"
+      found_diff = false
+      hist = []
+
+      found_count = 0
+      j.times do |i|
+        found_diff ||= (i >= lines1.size || i >= lines2.size || lines1[i] != lines2[i])
+        s = sprintf("%3d:",i)
+        if !found_diff
+          hist << "#{s}  #{lines1[i]}\n      #{lines2[i]}\n"
+        else
+          if found_count < 3
+            if i < lines1.size
+              s << "  #{lines1[i]}\n"
+            else
+              s << "  ---END---\n"
+            end
+            if i < lines2.size
+              s << "      #{lines2[i]}\n"
+            else
+              s << "      ---END---\n"
+            end
+            hist << s
+          end
+          found_count += 1
+        end
+        while hist.size > 6
+          hist.shift
+        end
+      end
+      dash = "-" * 95 + "\n"
+      raise IllegalStateException,"output did not match expected:\n#{dash}#{hist.join('')}#{dash}"
+    end
+  end
 end
-  
 
 # Convenience method to detect if a script is being run
 # e.g. as a 'main' method (for debug purposes only).
@@ -328,14 +593,6 @@ if defined? Test::Unit
       method_setup
     end
     
-    def out_dir
-      "_output_"
-    end
-    
-    def out_path(f)
-      File.join(out_dir,f)
-    end
-    
     def teardown
       if _suite_active?
         if !defined? @@suiteSetup
@@ -360,3 +617,136 @@ if defined? Test::Unit
   end
 end
 
+# Construct a string from an array of bytes 
+# @param byte_array array of bytes, or string (in which case it
+#   returns it unchanged)
+#
+def bytes_to_str(byte_array)
+  return byte_array if byte_array.is_a? String
+  
+  byte_array.pack('C*')
+end
+
+# Construct an array of bytes from a string  
+# @param str string, or array of bytes (in which case it
+#   returns it unchanged)
+#
+def str_to_bytes(str)
+  return str if str.is_a? Array
+  str.bytes
+end
+
+# Get directory entries, excluding '.' and '..'
+#
+def dir_entries(path)
+  ents = Dir.entries(path)
+  ents.reject!{|entry| entry == '.' || entry == '..'}
+end
+
+def int_to_bytes(x)
+  [(x >> 24) & 0xff, (x >> 16) & 0xff, (x >> 8) & 0xff, x & 0xff]
+end
+  
+def short_to_bytes(x) 
+  [(x >> 8) & 0xff, x & 0xff]
+end
+ 
+# Decode a short from an array of bytes (big-endian).
+# @param ba array of bytes
+# @param offset offset of first (most significant) byte
+#
+def short_from_bytes(ba, offset=0) 
+  (ba[offset] << 8) | ba[offset + 1] 
+end
+    
+# Decode an int from an array of bytes (big-endian).
+# @param ba array of bytes
+# @param offset offset of first (most significant) byte
+#
+def int_from_bytes(ba, offset=0) 
+  (((((ba[offset] << 8) | ba[offset + 1]) << 8) | \
+      ba[offset + 2]) << 8) | ba[offset + 3]
+end
+
+# Transform string to 8-bit ASCII (i.e., just treat each byte as-is)
+#
+def to_ascii8(str)
+  str.force_encoding("ASCII-8BIT")
+end
+
+# Verify that a string is encoded as ASCII-8BIT
+def simple_str(s)
+  if s.encoding.name != 'ASCII-8BIT' && s.encoding.name != 'UTF-8'
+    pr("string [%s]\n encoding is %s,\n expected ASCII-8BIT\n",s,s.encoding.name)
+    assert!(false)
+  end
+end
+
+# Truncate or pad string so it has a particular size
+#
+# @param s input string
+# @param size 
+# @param pad padding character to use if string needs to grow
+# @return modified string
+#
+def str_sized(s, size, pad="\0")
+  s[0...size].ljust(size,pad)
+end
+
+# Determine if running on the Windows operating system.
+# Note: there is some debate about the best way to do this.
+#
+def windows?
+  if !defined? $__windows__
+    $__windows__ = (RUBY_PLATFORM =~ /mswin/)
+  end
+  $__windows__
+end
+
+# Mark all constants ending with '_' as private constants
+#
+# @param entity the class to examine
+# @param add_non_suffix_versions if true, for each constant ABC_ found, also
+#    defines a constant ABC with the same value that is also private
+#
+def privatize(entity, add_non_suffix_versions = false)
+  
+  db = false
+  
+  # First command defines constants ABC = n for each constant ABC_ = n;
+  # Second declares both versions to be private
+  
+  cmd1 = nil
+  cmd2 = nil
+  
+  entity.constants.each do |c|
+    nm = c.to_s
+    
+    if nm.end_with?('_')
+      nm_small = nm[0..-2]
+      
+      if !cmd2
+        if add_non_suffix_versions
+          cmd1 = ''
+        end
+        cmd2 = 'private_constant '
+      else
+        cmd2 << ','
+      end 
+      
+      
+      !cmd1 || cmd1 << entity.to_s << '::' << nm_small << '=' << entity.const_get(c).to_s << "\n"
+      !cmd1 || cmd2 << ':' << nm_small << ','
+      cmd2 << ':' << nm
+    end
+  end
+  
+  if cmd2
+     if cmd1
+       !db || pr("about to eval:\n%s\n",cmd1)
+       eval(cmd1)
+     end
+     !db || pr("about to eval:\n%s\n",cmd2)
+     eval(cmd2)
+  end
+end

data/test/_misc_/sampletokens_dfa.txt

@@ -0,0 +1 @@
+{"version":1.0,"tokens":["WS","DBL","INT","LBL","ID","ASSIGN","EQUIV","IF","DO","BROP","BRCL"],"states":[[false,[[[125,126],1],[[123,124],2],[[100,101],3],[[105,106],4],[[61,62],5],[[65,91,95,96,97,100,101,105,106,123],6],[[39,40],7],[[48,58],8],[[45,46],9],[[46,47],10],[[9,11,12,13,32,33,92,93],11],[[47,48],12]]],[false,[[[-12,-11],14]]],[false,[[[-11,-10],14]]],[false,[[[48,58,65,91,95,96,97,111,112,123],6],[[-6,-5],14],[[111,112],22]]],[false,[[[48,58,65,91,95,96,97,102,103,123],6],[[-6,-5],14],[[102,103],21]]],[false,[[[-7,-6],14],[[61,62],20]]],[false,[[[48,58,65,91,95,96,97,123],6],[[-6,-5],14]]],[false,[[[0,10,11,39,40,92,93,1114112],7],[[39,40],17],[[92,93],18]]],[false,[[[48,58],8],[[46,47],10],[[-4,-3],14]]],[false,[[[48,58],8],[[46,47],10]]],[false,[[[48,58],16]]],[false,[[[9,11,12,13,32,33,92,93],11],[[-2,-1],14]]],[false,[[[47,48],13]]],[false,[[[0,10,11,1114112],13],[[-2,-1],14],[[10,11],15]]],[true,[]],[false,[[[-2,-1],14]]],[false,[[[-3,-2],14],[[48,58],16]]],[false,[[[-5,-4],14]]],[false,[[[0,10,11,39,40,92,93,1114112],7],[[92,93],18],[[39,40],19]]],[false,[[[0,10,11,39,40,92,93,1114112],7],[[-5,-4],14],[[39,40],17],[[92,93],18]]],[false,[[[-8,-7],14]]],[false,[[[48,58,65,91,95,96,97,123],6],[[-9,-8],14]]],[false,[[[48,58,65,91,95,96,97,123],6],[[-10,-9],14]]]]}

data/test/_output_270_filter_ws.txt

@@ -0,0 +1,26 @@
+(line 1, col 1)   : speed
+(line 1, col 6)   : =
+(line 1, col 7)   : 42
+(line 1, col 9)   : gravity
+(line 1, col 16)  : =
+(line 1, col 17)  : -9.80
+ ...couldn't find sequence...
+(line 1, col 22)  : {
+(line 1, col 23)  : color
+(line 1, col 29)  : =
+(line 1, col 30)  : green
+(line 1, col 35)  : }
+(line 1, col 36)  : title
+(line 1, col 41)  : =
+(line 1, col 42)  : 'This is a string with \' an escaped delimiter'
+(line 1, col 89)  : if
+(line 1, col 91)  : gravity
+(line 1, col 98)  : ==
+(line 1, col 100) : 12
+ ...read BROP DO ID sequence...
+   (line 1, col 102) : {
+   (line 1, col 103) : do
+   (line 1, col 105) : something
+   (line 1, col 114) : }
+(line 1, col 115) : do
+(line 1, col 117) : something_else

data/test/test_tokn.rb

@@ -3,11 +3,8 @@ require 'test/unit'
 require_relative '../lib/tokn/tools.rb'
 req('range_partition dfa dfa_builder tokenizer token_defn_parser')
 
-# Get access to Tokn namespace
+#SINGLETEST = "test_100_add"
 
-#setTestDir()
-
-#SINGLETEST = "test_ps_output_multi"
 if defined? SINGLETEST
   if main?(__FILE__)
     ARGV.concat("-n  #{SINGLETEST}".split)
@@ -18,29 +15,27 @@ class TestTokn <  MyTestSuite
 
   include Tokn, ToknInternal
   
-#  def data_file(f)
-#    File.join("data",f)
-#    #    File.dirname(__FILE__)+"/data/"+f
-#  end
-
   def suite_setup
-
+    
     # Make current directory = the one containing this script
     main?(__FILE__)
 
-    if !File.directory?(out_dir)
-      Dir.mkdir(out_dir)
+    @@out_dir = File.absolute_path(File.join(Dir.pwd,"_misc_"))
+      
+    if !File.directory?(@@out_dir)
+      Dir.mkdir(@@out_dir)
     end
     
-    @@sampleText = readTextFile("sampletext.txt")
-    @@sampleTokens = readTextFile("sampletokens.txt")
+    @@sampleText = read_text_file("sampletext.txt")
+    @@sampleTokens = read_text_file("sampletokens.txt")
   end
 
-  #  def withinTestDir(f)
-  #    File.join(@@testDir,"__source__")
-
+  def out_path(f)
+    File.join(@@out_dir,f)
+  end
+  
   def suite_teardown
-    remove_file_or_dir(out_dir)
+#    remove_file_or_dir(@@out_dir)
   end
 
   def method_setup
@@ -332,7 +327,7 @@ class TestTokn <  MyTestSuite
     addset(40,50)
     @par.prepare
 
-    @par.generatePDF(out_dir)
+    @par.generatePDF(@@out_dir)
 
     prep
     add 25,33
@@ -367,13 +362,13 @@ END
     s = x.startState
     x.endState.finalState = true
 
-    s.generatePDF(out_dir,"nfa")
+    s.generatePDF(@@out_dir,"nfa")
 
     r = s.reverseNFA()
-    r.generatePDF(out_dir,"reversed")
+    r.generatePDF(@@out_dir,"reversed")
 
     dfa = DFABuilder.nfa_to_dfa(s)
-    dfa.generatePDF(out_dir,"buildDFA")
+    dfa.generatePDF(@@out_dir,"buildDFA")
   end
 
   def test_180_cvt_NFA_to_DFA
@@ -382,14 +377,14 @@ END
     s = x.startState
     x.endState.finalState = true
 
-    s.generatePDF(out_dir,"nfa")
+    s.generatePDF(@@out_dir,"nfa")
 
     dfa = DFABuilder.nfa_to_dfa(s)
-    dfa.generatePDF(out_dir,"dfa")
+    dfa.generatePDF(@@out_dir,"dfa")
 
     oldToNewMap, maxId2 = dfa.duplicateNFA(42)
     dfa2 = oldToNewMap[dfa]
-    dfa2.generatePDF(out_dir,"dfa_duplicated")
+    dfa2.generatePDF(@@out_dir,"dfa_duplicated")
   end
 
   def test_190_TokenDefParser
@@ -399,7 +394,7 @@ END
     td = TokenDefParser.new(s)
 
     tokDFA = td.dfa
-    tokDFA.startState.generatePDF(out_dir,"TokenDFA")
+    tokDFA.startState.generatePDF(@@out_dir,"TokenDFA")
 
   end
 
@@ -525,10 +520,11 @@ END
 
   def test_270_filter_ws
 
+    capture_begin
+    
     dfa = DFA.from_script_file("sampletokens.txt")
-    t = Tokenizer.new(dfa,  readTextFile("sampletext.txt"), "WS")
+    t = Tokenizer.new(dfa,  read_text_file("sampletext.txt"), "WS")
 
-    s = ''
     while t.hasNext do
       
       tk = t.peek
@@ -536,48 +532,19 @@ END
       if t.nameOf(tk) == 'BROP'
         lst = t.readSequenceIf('BROP DO ID BRCL')
         if lst
-          s << " ...read BROP DO ID sequence...\n"  
-          lst.each{ |x| s << "   #{d(x)}\n"}
+          puts " ...read BROP DO ID sequence..."  
+          lst.each{ |x| puts "   #{d(x)}"}
           next
         else
-          s << " ...couldn't find sequence...\n"  
+          puts " ...couldn't find sequence..."  
         end
       end
       
       tk = t.read
-      s << d(tk) << "\n"
-      
+      puts d(tk)  
     end
-    exp =<<"EXP"
-(line 1, col 1)   : speed
-(line 1, col 6)   : =
-(line 1, col 7)   : 42
-(line 1, col 9)   : gravity
-(line 1, col 16)  : =
-(line 1, col 17)  : -9.80
- ...couldn't find sequence...
-(line 1, col 22)  : {
-(line 1, col 23)  : color
-(line 1, col 29)  : =
-(line 1, col 30)  : green
-(line 1, col 35)  : }
-(line 1, col 36)  : title
-(line 1, col 41)  : =
-(line 1, col 42)  : 'This is a string with \\' an escaped delimiter'
-(line 1, col 89)  : if
-(line 1, col 91)  : gravity
-(line 1, col 98)  : ==
-(line 1, col 100) : 12
- ...read BROP DO ID sequence...
-   (line 1, col 102) : {
-   (line 1, col 103) : do
-   (line 1, col 105) : something
-   (line 1, col 114) : }
-(line 1, col 115) : do
-(line 1, col 117) : something_else
-EXP
-
-   assert(s.strip == exp.strip)
+    
+    match_expected_output
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokn
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Jeff Sember
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-09 00:00:00.000000000 Z
+date: 2013-04-15 00:00:00.000000000 Z
 dependencies: []
 description: "Given a script containing token descriptions (each a regular expression),
   \ntokn compiles an automaton which it can then use to efficiently convert a \ntext
@@ -40,6 +40,15 @@ files:
 - test/_OLD_/data/sampletokens_dfa.txt
 - test/_OLD_/test.rb
 - test/_OLD_/testcmds
+- test/_misc_/__mygraph__TokenDFA.pdf
+- test/_misc_/__mygraph__buildDFA.pdf
+- test/_misc_/__mygraph__dfa.pdf
+- test/_misc_/__mygraph__dfa_duplicated.pdf
+- test/_misc_/__mygraph__nfa.pdf
+- test/_misc_/__mygraph__partition.pdf
+- test/_misc_/__mygraph__reversed.pdf
+- test/_misc_/sampletokens_dfa.txt
+- test/_output_270_filter_ws.txt
 - test/sampletext.txt
 - test/sampletokens.txt
 - test/test_tokn.rb