langscan 1.2

data/tasks/extconf.rake

@@ -0,0 +1,13 @@
+namespace :extconf do
+  desc "Compiles the Ruby extension"
+  task :compile
+end
+
+task :compile => "extconf:compile"
+
+task :test => :compile
+
+BIN = "*.{bundle,jar,so,obj,pdb,lib,def,exp}"
+$hoe.clean_globs |= ["ext/**/#{BIN}", "lib/**/#{BIN}", 'ext/**/Makefile']
+$hoe.spec.require_paths = Dir['{lib,ext/*}']
+$hoe.spec.extensions = FileList["ext/**/extconf.rb"].to_a

data/tasks/extconf/langscan.rake

@@ -0,0 +1,42 @@
+namespace :extconf do
+  extension = File.basename(__FILE__, '.rake')
+  ext_confs = Dir.glob("**/extconf.rb")
+
+  task :compile => :compile_all do
+    if Dir.glob("**/*.{o,so,dll,bundle}").length == 0
+      STDERR.puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      STDERR.puts "Gem actually failed to build.  Your system is"
+      STDERR.puts "NOT configured properly to build langscan."
+      STDERR.puts "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+      exit(1)
+    end
+  end
+
+  task :compile_all do
+    oks = []
+    ngs = []
+    ext_confs.each do |ext_conf|
+      _ext = File.dirname(ext_conf)
+      lang_name = File.basename(_ext)
+      Dir.chdir(_ext) do
+        ruby "extconf.rb"
+        sh(PLATFORM =~ /win32/ ? 'nmake' : 'make') do |ok, res|
+          if !ok
+            require "fileutils"
+            FileUtils.rm Dir.glob('*.{so,o,dll,bundle}')
+            ngs << lang_name
+          else
+            oks << lang_name
+          end
+        end
+      end
+    end
+    oks.each do |lang_name|
+      STDOUT.puts "#{lang_name}: SUCCESS"
+    end
+    ngs.each do |lang_name|
+      STDERR.puts "#{lang_name}: FAILED"
+    end
+  end
+
+end

data/test/langscan/brainfuck/test/test_scan.rb

@@ -0,0 +1,55 @@
+require 'test/unit'
+require 'langscan/brainfuck'
+
+class TestScan < Test::Unit::TestCase
+  def assert_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      yielded = true
+      yield(*block_args)
+    }
+    assert(yielded, "block not yielded")
+  end
+  
+  def assert_not_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      assert(false, "block yielded")
+    }
+  end
+  
+  def assert_yield_any(recv, meth, *args)
+    success = false
+    recv.__send__(meth, *args) {|*block_args|
+      if yield(*block_args)
+        assert(true)
+        success = true
+        break
+      end
+    }
+    assert(false, "no expected yields") unless success
+  end
+  
+  def assert_yield_all(recv, meth, *args)
+    recv.__send__(meth, *args) {|*block_args|
+      if !yield(*block_args)
+        assert(false, "unexpected yields")
+        return
+      end
+    }
+    assert(true)
+  end
+  
+  def test_scan
+  	assert_yield_all(LangScan::Brainfuck, :scan, "+-.,<>[]") {|f|
+		f.type == :ident && f.text.length==1
+	}
+  	assert_yield_all(LangScan::Brainfuck, :scan, "+++") {|f|
+		f.type == :ident && f.text.length==1
+	}
+  	assert_yield_all(LangScan::Brainfuck, :scan, "I love you") {|f|
+		f.type == :comment
+	}
+  end
+end
+

data/test/langscan/c/test/test_scan.rb

@@ -0,0 +1,216 @@
+require 'test/unit'
+require 'langscan/c'
+
+class TestScan < Test::Unit::TestCase
+  def assert_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      yielded = true
+      yield(*block_args)
+    }
+    assert(yielded, "block not yielded")
+  end
+
+  def assert_not_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      assert(false, "block yielded")
+    }
+  end
+
+  def test_token_position
+    assert_yield(LangScan::C, :scan, "a(1,2,3)") {|f|
+      next if f.type != :funcall
+      assert_equal(0, f.beg_byteno)
+      #assert_equal(1, f.end_byteno)
+    }
+  end
+
+  def assert_yield_any(recv, meth, *args)
+    success = false
+    recv.__send__(meth, *args) {|*block_args|
+      if yield(*block_args)
+        assert(true)
+        success = true
+        break
+      end
+    }
+    assert(false, "no expected yields") unless success
+  end
+
+  def assert_yield_all(recv, meth, *args)
+    recv.__send__(meth, *args) {|*block_args|
+      if !yield(*block_args)
+        assert(false, "unexpected yields")
+        return
+      end
+    }
+    assert(true)
+  end
+
+  def test_name
+    assert_yield_any(LangScan::C, :scan, "a()") {|f|
+      f.type == :funcall && f.text == 'a'
+    }
+  end
+
+  def test_macro
+    assert_yield_any(LangScan::C, :scan, "#define a() b") {|f|
+      f.type == :fundef
+    }
+    assert_yield_all(LangScan::C, :scan, "#define a () b") {|f|
+      f.type != :fundef
+    }
+  end
+
+  def test_funtype
+    assert_yield_all(LangScan::C, :scan, "int fun(type (*)())") {|f|
+      !(f.type == :fundef || f.type == :funcall || f.type == :fundecl) ||
+      f.text == 'fun'
+    }
+  end
+
+  def test_decl
+    assert_yield_any(LangScan::C, :scan, "int f();") {|f|
+      f.type == :fundecl && f.text == 'f'
+    }
+  end
+
+  def test_preproc_defined
+    assert_yield_all(LangScan::C, :scan, "#if defined(MACRO)\n") {|f|
+      !(f.type == :funcall && f.text == 'defined')
+    }
+  end
+
+  def test_fragment_and_function
+    regions = {}
+    LangScan::C.scan("a()") {|t|
+      f = t
+      f = f.name_token if f.respond_to? :name_token
+      r = f.beg_byteno
+      assert(!regions.include?(r), "duplicate token: #{regions[r].inspect} and #{t.inspect}")
+      regions[r] = t
+    }
+  end
+
+  def test_kandr_portability_fundecl
+    src = "void rbuf_initialize PARAMS ((struct rbuf *, int));"
+    result = []
+    assert_yield(LangScan::C, :scan, src) {|f|
+      if f.type == :fundecl && f.text == 'rbuf_initialize'
+        result << :rbuf_initialize
+      end
+      if f.type == :funcall && f.text == 'PARAMS'
+        result << :PARAMS
+      end
+    }
+    assert_equal([:rbuf_initialize], result)
+  end
+
+  def test_funcdef
+    [
+      "int fun() {}",
+      "int fun(void) {}",
+      "int fun(int arg) {}",
+      "int fun(arg) int arg; {}",
+      "int fun(arg) struct tag *arg; {}",
+      "int fun(arg) typedefed_ident arg; {}",
+    ].each {|src|
+      result = false
+      assert_yield(LangScan::C, :scan, src) {|f|
+        if f.type == :fundef && f.text == 'fun'
+          result = true
+        end
+      }
+      assert(result, src)
+    }
+  end
+
+  def test_extern_c
+    src = 'extern "C" { void fun(); }'
+    assert_yield_any(LangScan::C, :scan, src) {|f|
+      f.type == :fundecl && f.text == 'fun'
+    }
+  end
+
+  def test_keyword
+    assert_yield_all(LangScan::C, :scan, "int()") {|f|
+      f.type != :funcall
+    }
+  end
+
+  def test_toplevel_comma
+    assert_yield(LangScan::C, :scan, ",") {|f|
+      assert_equal(LangScan::Fragment, f.class)
+      assert_equal(:punct, f.type)
+      assert_equal(",", f.text)
+    }
+  end
+
+  def test_funcall
+    result = []
+    LangScan::C.scan("f(){g();}") {|f|
+      next unless f.type == :fundef || f.type == :funcall
+      result << [f.type, f.text]
+    }
+    assert_equal([[:fundef, 'f'], [:funcall, 'g']], result)
+  end
+
+  # C++
+
+  def assert_fragment_type(type, text, src)
+    found = false
+    LangScan::C.scan(src) {|f|
+      if f.text == text
+        if found
+          raise "token #{text} occurred twice"
+        else
+          found = true
+          assert_equal(type, f.type, "fragment type of #{text}")
+        end
+      end
+    }
+    unless found
+      raise "token #{text} not found"
+    end
+  end
+
+  def test_class
+    assert_fragment_type(:classdef, 'c', "class c {};")
+    assert_fragment_type(:classdef, 'c1', "class c1 : c2 {};")
+    assert_fragment_type(:classref, 'c2', "class c1 : c2 {};") 
+    assert_fragment_type(:classdecl, 'c3', "class c3;")
+  end
+
+  def test_struct
+    assert_fragment_type(:classdef, 'c', "struct c {};")
+    assert_fragment_type(:classdef, 'c1', "struct c1 : c2 {};")
+    assert_fragment_type(:classref, 'c2', "struct c1 : c2 {};") 
+    assert_fragment_type(:classdecl, 'c3', "struct c3;")
+  end
+
+  def test_struct_2
+    src = 'void f(void) { struct s v; return; }'
+    assert_fragment_type(:classref, 's', src)
+    assert_fragment_type(:keyword, 'return', src)
+  end
+
+  def check_scan(src)
+    LangScan::C.scan(src) {|f|
+      assert_equal(f.text, src[f.beg_byteno...f.end_byteno])
+    }
+  end
+
+  def test_sharp_in_non_initial_state
+    check_scan("struct x\n#;")
+  end
+
+  def test_invalid_escape_sequence
+    assert_fragment_type(:string, '"\w"', '"\w"')
+    assert_fragment_type(:string, '"foo\bar.h"', '#include "foo\bar.h"')
+  end
+
+  def test_fundef_returns_user_defined_type
+    assert_fragment_type(:fundef, 'foo', 'VALUE foo() {}')
+  end
+end

data/test/langscan/c/test/test_token.rb

@@ -0,0 +1,41 @@
+require 'test/unit'
+require 'langscan/c'
+
+class TestToken < Test::Unit::TestCase
+  def test_comment
+    t = LangScan::C::Tokenizer.new("/* * */")
+    type, text = t.get_token
+    assert_equal("/* * */", text)
+    assert_equal(:comment, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::C::Tokenizer.new("/* **/")
+    type, text = t.get_token
+    assert_equal("/* **/", text)
+    assert_equal(:comment, type)
+    assert_equal(nil, t.get_token)
+  end
+
+  def test_c99_comment
+    t = LangScan::C::Tokenizer.new("// abc")
+    type, text = t.get_token
+    assert_equal("// abc", text)
+    assert_equal(:comment, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::C::Tokenizer.new("//")
+    type, text = t.get_token
+    assert_equal("//", text)
+    assert_equal(:comment, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::C::Tokenizer.new("//def \n")
+    type, text = t.get_token
+    assert_equal("//def ", text)
+    assert_equal(:comment, type)
+    type, text = t.get_token
+    assert_equal("\n", text)
+    assert_equal(:space, type)
+    assert_equal(nil, t.get_token)
+  end
+end

data/test/langscan/csharp/test/test_scan.rb

@@ -0,0 +1,157 @@
+require 'test/unit'
+require 'langscan/csharp'
+
+class TestCsharp < Test::Unit::TestCase
+  def assert_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      yielded = true
+      yield(*block_args)
+    }
+    assert(yielded, "block not yielded")
+  end
+
+  def assert_not_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      assert(false, "block yielded")
+    }
+  end
+
+  def test_token_position
+    assert_yield(LangScan::Csharp, :scan, "a(1,2,3)") {|f|
+      next if f.type != :funcall
+      assert_equal(0, f.beg_byteno)
+    }
+  end
+
+  def assert_yield_any(recv, meth, *args)
+    success = false
+    recv.__send__(meth, *args) {|*block_args|
+      if yield(*block_args)
+        assert(true)
+        success = true
+        break
+      end
+    }
+    assert(false, "no expected yields") unless success
+  end
+
+  def assert_yield_all(recv, meth, *args)
+    recv.__send__(meth, *args) {|*block_args|
+      if !yield(*block_args)
+        assert(false, "unexpected yields")
+        return
+      end
+    }
+    assert(true)
+  end
+
+  def test_name
+    assert_yield_any(LangScan::Csharp, :scan, "a()") {|f|
+      f.type == :funcall && f.text == 'a'
+    }
+  end
+
+
+  def test_delegate
+    assert_yield_any(LangScan::Csharp, :scan, "delegate int f();") {|f|
+      f.type == :ident && f.text == 'f'
+    }
+  end
+
+
+  def test_fragment_and_function
+    regions = {}
+    LangScan::Csharp.scan("a()") {|t|
+      f = t
+      f = f.name_token if f.respond_to? :name_token
+      r = f.beg_byteno
+      assert(!regions.include?(r), "duplicate token: #{regions[r].inspect} and #{t.inspect}")
+      regions[r] = t
+    }
+  end
+
+  def test_funcdef
+    [
+      "int fun() {}",
+      "int fun(int arg) {}",
+      "public static void fun() {}",
+      "int fun() /* comment */ {}",
+      "int fun() // comment\n {}",
+    ].each {|src|
+      result = false
+      assert_yield(LangScan::Csharp, :scan, src) {|f|
+        if f.type == :fundef && f.text == 'fun'
+          result = true
+        end
+      }
+      assert(result, src)
+    }
+  end
+
+  def test_keyword
+    assert_yield_all(LangScan::Csharp, :scan, "int()") {|f|
+      f.type != :funcall
+    }
+  end
+
+  def test_funcall
+    result = []
+    LangScan::Csharp.scan("f(){g();}") {|f|
+      next unless f.type == :fundef || f.type == :funcall
+      result << [f.type, f.text]
+    }
+    assert_equal([[:fundef, 'f'], [:funcall, 'g']], result)
+  end
+
+  def assert_fragment_type(type, text, src)
+    found = false
+    LangScan::Csharp.scan(src) {|f|
+      if f.text == text
+        if found
+          raise "token #{text} occurred twice"
+        else
+          found = true
+          assert_equal(type, f.type, "fragment type of #{text}")
+        end
+      end
+    }
+    unless found
+      raise "token #{text} not found"
+    end
+  end
+
+  def test_class
+    assert_fragment_type(:classdef, 'c', "class c {};")
+    assert_fragment_type(:classdef, 'c1', "class c1 : c2 {};")
+    assert_fragment_type(:classref, 'c2', "class c1 : c2 {};") 
+    assert_fragment_type(:classref, 'c4', "class c1 : c2.c3.c4 {};") 
+  end
+
+  def test_struct
+    assert_fragment_type(:classdef, 'c', "struct c {};")
+    assert_fragment_type(:classdef, 'c1', "struct c1 : c2 {};")
+    assert_fragment_type(:classref, 'c2', "struct c1 : c2 {};") 
+  end
+
+  def check_scan(src)
+    LangScan::Csharp.scan(src) {|f|
+      assert_equal(f.text, src[f.beg_byteno...f.end_byteno])
+    }
+  end
+
+  def test_sharp_in_non_initial_state
+    check_scan("struct x\n#;")
+  end
+
+  def test_invalid_escape_sequence
+    assert_fragment_type(:string, '"\w"', '"\w"')
+    assert_fragment_type(:string, '"foo\bar.h"', 'str = "foo\bar.h"')
+  end
+
+  def test_fundef_returns_user_defined_type
+    assert_fragment_type(:fundef, 'foo', 'VALUE foo() {}')
+  end
+
+end