tokn 0.0.4

data/test/test.rb

@@ -0,0 +1,519 @@
+require 'test/unit'
+require_relative '../lib/tokn/tools.rb'
+req('range_partition dfa dfa_builder tokenizer')
+
+
+def dataPath(f)
+  File.dirname(__FILE__)+"/data/"+f
+end
+
+setTestDir()
+
+# Various unit tests for state machines, character range sets, etc.
+#
+class TestComponent < Test::Unit::TestCase
+  
+  SKIPMOST = false  # skip most of the tests?
+  
+  def add(lower, upper = nil)
+    @cs.add(lower,upper)
+  end
+
+  def remove(lower, upper = nil)
+    @cs.remove(lower,upper)
+  end
+
+  def swap
+    @ct = @cs
+    prep
+  end
+  
+  def isect
+    @cs.intersect!(@ct)
+  end
+  
+  def diff
+    @cs.difference!(@ct)
+  end
+  
+  def equ(s, arr = nil)
+    arr ||= @cs.array
+    ia = s.split.map{|n| n.to_i}
+    assert_equal(ia,arr)
+  end
+  
+  def test_add
+    prep
+    
+    add(72,81)
+    equ '72 81' 
+    
+    add(50)
+    equ '50 51 72 81' 
+
+    add(75,77)
+    equ '50 51 72 81' 
+     
+    add(72,78)
+    equ '50 51 72 81' 
+    
+    add(70,78)
+    equ '50 51 70 81' 
+
+    add 60 
+    equ '50 51 60 61 70 81' 
+     
+    add 40
+    equ '40 41 50 51 60 61 70 81' 
+
+    add 41     
+    equ '40 42 50 51 60 61 70 81' 
+    
+    add 81
+    equ '40 42 50 51 60 61 70 82' 
+    
+    add 83
+    equ '40 42 50 51 60 61 70 82 83 84' 
+    
+    add 49,84
+    equ '40 42 49 84'
+
+    add 39,86
+    equ '39 86'
+  end
+  
+  def test_intersect
+    prep
+    add 39,86
+    swap
+    add 50,70
+    isect
+    equ '50 70'
+    
+    swap
+    add 20,25
+    add 35,51
+    add 62,68
+    add 72,80
+    isect
+    equ '50 51 62 68'
+     
+    prep
+    swap
+    add 50,70
+    isect
+    equ ''
+    
+    add 50,70
+    swap
+    add 50,70
+    isect
+    equ '50 70'
+    
+    
+    prep
+    add 20,25
+    swap
+    add 25,30
+    isect
+    equ ''
+    
+  end
+ 
+  def test_difference
+    prep
+    add 20,30
+    add 40,50
+    swap
+    
+    
+    add 20,80
+    diff
+    equ '30 40 50 80'
+    
+    prep
+    add 19,32
+    diff
+    equ '19 20 30 32'
+    
+    prep
+    add 30,40
+    diff
+    equ '30 40'
+    
+    prep
+    add 20,30
+    add 40,50
+    diff
+    equ ''
+    
+    prep
+    add 19,30
+    add 40,50
+    diff
+    equ '19 20'
+
+    prep
+    add 20,30
+    add 40,51
+    diff
+    equ '50 51'
+    
+    
+  end
+ 
+  def prep
+    @cs = CodeSet.new
+  end
+  
+  def test_illegalRange
+    prep
+    
+    assert_raise(RangeError) { add 60,50 }
+    assert_raise(RangeError) { add 60,60 }
+  end
+  
+  def neg lower, upper
+    @cs.negate lower, upper
+  end
+  
+  def test_negate
+    prep
+    add 10,15
+    add 20,25
+    add 30
+    add 40,45
+    equ '10 15 20 25 30 31 40 45'
+    neg 22,37 
+    equ '10 15 20 22 25 30 31 37 40 45'
+    neg 25,27 
+    equ '10 15 20 22 27 30 31 37 40 45'
+    neg 15,20 
+    equ '10 22 27 30 31 37 40 45'
+    
+    prep
+    add 10,22
+    @cs.negate
+    equ '0 10 22 1114112'
+    
+    
+    prep
+    add 10,20
+    neg 10,20
+    equ ''
+
+    prep
+    add 10,20
+    add 30,40
+    neg 5,10
+    equ '5 20 30 40'
+
+    prep
+    add 10,20
+    add 30,40
+    neg 25,30
+    equ '10 20 25 40'
+
+    prep
+    add 10,20
+    add 30,40
+    neg 40,50
+    equ '10 20 30 50'
+
+    prep
+    add 10,20
+    add 30,40
+    neg 41,50
+    equ '10 20 30 40 41 50'
+
+    prep
+    add 10,20
+    add 30,40
+    neg 15,35
+    equ '10 15 20 30 35 40'
+  end
+
+  def test_remove
+
+    prep
+    add 10,20
+    add 30,40
+    remove 29,41
+    equ '10 20'
+
+    add 30,40
+    equ '10 20 30 40'
+    
+    remove 20,30
+    equ '10 20 30 40'
+    
+    remove 15,35
+    equ '10 15 35 40'
+    
+    remove 10,15
+    equ '35 40'
+    remove 35 
+    equ '36 40'
+    remove 40
+    equ '36 40'
+    remove 38
+    equ '36 38 39 40'
+    remove 37,39
+    equ '36 37 39 40'
+    
+  end
+  
+  
+  def dset(st)
+    s = ''
+    st.each{|x| 
+      if s.length > 0 
+        s+= ' '
+      end
+      s += d(x)
+    }
+    return s
+  end
+  
+  
+  def newpar
+    @par = RangePartition.new
+  end
+  
+  def addset(lower, upper = nil)
+    upper ||= lower + 1
+    r = CodeSet.new(lower,upper)
+    @par.addSet(r)  
+  end
+  
+  def apply
+    list = @par.apply(@cs)
+    res = []
+    list.each do |x|
+      res.concat x.array
+    end
+    @parResult = res
+  end
+  
+  def test_partition
+    return if SKIPMOST
+    
+    newpar
+    addset(20,30)
+    addset(25,33)
+    addset(37)
+    addset(40,50)
+    @par.prepare
+    
+    @par.generatePDF()
+    
+    prep
+    add 25,33
+    
+    apply
+    equ('25 30 30 33', @parResult)
+    
+    
+    prep
+    add 37
+    apply
+    equ('37 38', @parResult)
+
+    prep
+    add 40,50
+    apply
+    equ('40 50', @parResult)
+    
+  end
+  
+
+  REGEX_SCRIPT = "(\\-?[0-9]+)|[_a-zA-Z][_a-zA-Z0-9]*|333q"
+
+  TOKEN_SCRIPT2 = <<'END'
+      sep:  \s
+      tku:  a(a|b)*
+      tkv:  b(aa|b*)
+      tkw:  bbb
+END
+
+  
+  def test_buildDFA
+    return if SKIPMOST
+    
+    x = RegParse.new(REGEX_SCRIPT)
+    s = x.startState
+    x.endState.finalState = true
+
+    s.generatePDF("nfa")
+    
+    r = s.reverseNFA()
+    r.generatePDF("reversed")
+    
+    dfa = DFABuilder.nfa_to_dfa(s)
+    dfa.generatePDF("buildDFA")
+  end
+
+  def test_cvtNFAToDFA
+    return if SKIPMOST
+    
+    x = RegParse.new(REGEX_SCRIPT)
+    s = x.startState
+    x.endState.finalState = true
+    
+    s.generatePDF("nfa")
+    
+    dfa = DFABuilder.nfa_to_dfa(s)
+    dfa.generatePDF("dfa")
+    
+    oldToNewMap, maxId2 = dfa.duplicateNFA(42)
+    dfa2 = oldToNewMap[dfa]
+    dfa2.generatePDF("dfa_duplicated")
+  end  
+
+
+  def test_TokenDefParser
+    return if SKIPMOST
+    
+    s = TOKEN_SCRIPT2
+    
+    td = TokenDefParser.new(s)
+    
+    tokDFA = td.dfa
+    tokDFA.startState.generatePDF("TokenDFA")
+    
+  end
+  
+  
+  
+  @@sampleText = readTextFile(dataPath("sampletext.txt"))
+  @@sampleTokens = readTextFile(dataPath("sampletokens.txt"))
+  
+  def makeTok
+    dfa = DFA.dfa_from_script(@@sampleTokens)
+    Tokenizer.new(dfa, @@sampleText)
+  end
+  
+  
+  def test_Tokenizer
+    return if SKIPMOST
+    
+      tok = makeTok
+      
+      tokList = []
+      while tok.hasNext
+        t = tok.read
+        tokList.push(t)
+      end
+       
+      tok.unread(tokList.size)
+      
+      tokList.each do |t1|
+        tName = tok.nameOf(t1)
+        t2 = tok.read(tName)
+      end
+  end
+
+  
+  
+  
+  def test_TokenizerMissingExpected
+    return if SKIPMOST
+
+    assert_raise TokenizerException do
+      
+      tok = makeTok
+      
+      tok.read
+      tok.read
+      tok.read
+      tok.read
+      tok.read("signedint")
+    end
+     
+   end
+  
+  def test_CompileDFAToDisk
+    tokScript = @@sampleTokens
+    testText = @@sampleText
+    
+    destPath = withinTestDir("sampletokens_dfa.txt")
+    
+    if File.exist?(destPath)
+      File.delete(destPath)
+    end
+    assert(!File.exist?(destPath))
+  
+    dfa = DFA.dfa_from_script(tokScript, destPath)
+    assert(File.exist?(destPath))
+    
+    tok = Tokenizer.new(dfa,  testText)
+    
+  end
+  
+  
+  def prep2
+    testText = @@sampleText
+    dfa = DFA.dfa_from_file(withinTestDir("sampletokens_dfa.txt"))
+    tok = Tokenizer.new(dfa, testText)  
+  end
+  
+  def test_readAndUnread
+    tok = prep2  
+    unread = false
+    while tok.hasNext
+      t = tok.read
+      pr("Read  %-8s %s\n",tok.nameOf(t),d(t))
+      
+      if !unread && tok.nameOf(t) == "DO"
+        pr("  ...pushing back four tokens...\n")
+        tok.unread(4)
+        unread = true
+        pr("  ...and resuming...\n")
+      end
+    end
+  end
+  
+  def test_UnrecognizedToken
+    assert_raise TokenizerException do
+      tok = prep2
+      while tok.hasNext
+          t = tok.read
+          if tok.nameOf(t) == "DO"
+            tok.read("BRCL") # <== this should raise problem
+          end
+      end
+    end
+  end
+  
+  def test_ReadPastEnd
+   assert_raise TokenizerException do
+      tok = prep2
+      while tok.hasNext
+          t = tok.read
+      end
+      tok.read
+    end
+  end
+
+  def test_UnreadBeforeStart
+    
+    assert_raise TokenizerException do
+      tok = prep2
+      k = 0
+      while tok.hasNext
+          t = tok.read
+          k += 1
+          if k == 15
+            tok.unread(5)
+            tok.unread(7)
+            tok.read()
+            tok.unread(4)
+            tok.unread(3)
+          end
+      end
+      tok.read
+    end
+  end
+end
+
+

data/test/testcmds

@@ -0,0 +1,4 @@
+#!/bin/bash
+ruby tokncompile.rb < sampletokens.txt > __testdata__/compiled.txt
+ruby toknprocess.rb __testdata__/compiled.txt sampletext.txt
+

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: tokn
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Jeff Sember
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-07 00:00:00.000000000 Z
+dependencies: []
+description: 'Given a script containing token descriptions (each a regular expression),
+  tokn compiles a DFA which it then uses to efficiently extract a sequence of tokens
+  from source files. '
+email: jpsember@gmail.com
+executables:
+- tokncompile
+- toknprocess
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/tokn.rb
+- lib/tokn/code_set.rb
+- lib/tokn/dfa.rb
+- lib/tokn/dfa_builder.rb
+- lib/tokn/range_partition.rb
+- lib/tokn/reg_parse.rb
+- lib/tokn/state.rb
+- lib/tokn/token_defn_parser.rb
+- lib/tokn/tokenizer.rb
+- lib/tokn/tokn_const.rb
+- lib/tokn/tools.rb
+- bin/tokncompile
+- bin/toknprocess
+- README.txt
+- test/data/sampletext.txt
+- test/data/sampletokens.txt
+- test/simple.rb
+- test/test.rb
+- test/testcmds
+- figures/sample_dfa.pdf
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.2
+signing_key: 
+specification_version: 4
+summary: Extracts tokens from source files
+test_files:
+- test/simple.rb
+- test/test.rb
+has_rdoc: