langscan 1.2

data/test/langscan/css/test/test_css.rb

@@ -0,0 +1,79 @@
+$LOAD_PATH << "../../.." unless $LOAD_PATH.include?("../../..")
+require 'test/unit'
+require 'langscan/css'
+
+class TestCSS < Test::Unit::TestCase
+  def assert_fragments(expected, code)
+    result = []
+    LangScan::CSS.scan(code) {|f| result.push(f) }
+    fragments = []
+    result.each {|fragment|
+      type, text = fragment.to_a
+      unless type == :punct
+	fragments << [type, text]
+      end
+    }
+    assert_equal(expected, fragments)
+  end
+
+  def test_sample_css
+    code = <<"End"
+@import "another.css";
+
+h1 {
+  border: 1px solid #ccc;
+}
+
+div.day {
+  font: medium bold sans-serif;
+}
+End
+    expected = [[:keyword, "@import"],
+      [:string, "\"another.css\""],
+      [:fundef, "h1"],
+      [:keyword, "border"],
+      [:ident, "1px"],
+      [:ident, "solid"],
+      [:ident, "ccc"],
+      [:fundef, "div"],
+      [:fundef, "day"],
+      [:keyword, "font"],
+      [:ident, "medium"],
+      [:ident, "bold"],
+      [:ident, "sans-serif"]]
+    assert_fragments(expected, code)
+  end
+
+  def test_comment
+    assert_fragments([[:comment, "/*c*/"]], "/*c*/")
+  end
+
+  def test_string
+    assert_fragments([[:keyword, "url"], [:string, "(a)"]], "url(a)")
+  end
+
+  def test_keyword
+    assert_fragments([[:keyword, "@import"], [:string, "\"a.css\""]],
+		     "@import \"a.css\";")
+    assert_fragments([[:keyword, "!important"]], "!important")
+    assert_fragments([[:keyword, "! important"]], "! important")
+  end
+
+  def test_selector
+    assert_fragments([[:fundef, "p"]], "p{}")
+    assert_fragments([[:fundef, "c"]], ".c{}")
+    assert_fragments([[:fundef, "i"]], "#i{}")
+    assert_fragments([[:fundef, "p"], [:fundef, "c"]], "p.c{}")
+    assert_fragments([[:fundef, "p"], [:fundef, "b"]], "p b{}")
+    assert_fragments([[:fundef, "h1"], [:fundef, "h2"]], "h1,h2{}")
+    assert_fragments([[:fundef, "p"], [:fundef, "b"]], "p>b{}")
+    assert_fragments([[:fundef, "a"], [:fundef, "active"]], "a:active{}")
+  end
+
+  def test_property
+    assert_fragments([[:fundef, "p"], [:keyword, "b"], [:ident, "a"]],
+		     "p{b:a}")
+    assert_fragments([[:fundef, "p"], [:keyword, "border"], [:ident, "0"]],
+		     "p{border:0;}")
+  end
+end

data/test/langscan/d/test/test_scan.rb

@@ -0,0 +1,233 @@
+require 'test/unit'
+require 'langscan/d'
+
+class TestScanD < Test::Unit::TestCase
+  def assert_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      yielded = true
+      yield(*block_args)
+    }
+    assert(yielded, "block not yielded")
+  end
+
+  def assert_not_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      assert(false, "block yielded")
+    }
+  end
+
+  def test_token_position
+    assert_yield(LangScan::D, :scan, "a(1,2,3)") {|f|
+      next if f.type != :funcall
+      assert_equal(0, f.beg_byteno)
+      #assert_equal(1, f.end_byteno)
+    }
+  end
+
+  def assert_yield_any(recv, meth, *args)
+    success = false
+    recv.__send__(meth, *args) {|*block_args|
+      if yield(*block_args)
+        assert(true)
+        success = true
+        break
+      end
+    }
+    assert(false, "no expected yields") unless success
+  end
+
+  def assert_yield_all(recv, meth, *args)
+    recv.__send__(meth, *args) {|*block_args|
+      if !yield(*block_args)
+        assert(false, "unexpected yields")
+        return
+      end
+    }
+    assert(true)
+  end
+
+  def test_funtype
+    assert_yield_all(LangScan::D, :scan, "int fun(type (*)())") {|f|
+      !(f.type == :fundef || f.type == :funcall || f.type == :fundecl) ||
+      f.text == 'fun'
+    }
+  end
+
+  def test_decl
+    assert_yield_any(LangScan::D, :scan, "int f();") {|f|
+      f.type == :fundecl && f.text == 'f'
+    }
+  end
+
+  def test_preproc_defined
+    assert_yield_all(LangScan::D, :scan, "#if defined(MACRO)\n") {|f|
+      !(f.type == :funcall && f.text == 'defined')
+    }
+  end
+
+  def test_fragment_and_function
+    regions = {}
+    LangScan::D.scan("a()") {|t|
+      f = t
+      f = f.name_token if f.respond_to? :name_token
+      r = f.beg_byteno
+      assert(!regions.include?(r), "duplicate token: #{regions[r].inspect} and #{t.inspect}")
+      regions[r] = t
+    }
+  end
+
+  def test_funcdef
+    [
+      "int fun() {}",
+      "int fun(void) {}",
+      "int fun(int arg) {}",
+    ].each {|src|
+      result = false
+      assert_yield(LangScan::D, :scan, src) {|f|
+        if f.type == :fundef && f.text == 'fun'
+          result = true
+        end
+      }
+      assert(result, src)
+    }
+  end
+
+  def test_extern_c
+    src = 'extern "C" { void fun(); }'
+    assert_yield_any(LangScan::D, :scan, src) {|f|
+      f.type == :fundecl && f.text == 'fun'
+    }
+  end
+
+  def test_keyword
+    assert_yield_all(LangScan::D, :scan, "int()") {|f|
+      f.type != :funcall
+    }
+  end
+
+  def test_toplevel_comma
+    assert_yield(LangScan::D, :scan, ",") {|f|
+      assert_equal(LangScan::Fragment, f.class)
+      assert_equal(:punct, f.type)
+      assert_equal(",", f.text)
+    }
+  end
+
+  def test_funcall
+    result = []
+    LangScan::D.scan("f(){g();}") {|f|
+      next unless f.type == :fundef || f.type == :funcall
+      result << [f.type, f.text]
+    }
+    assert_equal([[:fundef, 'f'], [:funcall, 'g']], result)
+  end
+
+  # C++
+
+  def assert_fragment_type(type, text, src)
+    found = false
+    LangScan::D.scan(src) {|f|
+      if f.text == text
+        if found
+          raise "token #{text} occurred twice"
+        else
+          found = true
+          assert_equal(type, f.type, "fragment type of #{text}")
+        end
+      end
+    }
+    unless found
+      raise "token #{text} not found"
+    end
+  end
+
+  def test_class
+    assert_fragment_type(:classdef, 'c', "class c {};")
+    assert_fragment_type(:classdef, 'c1', "class c1 : c2 {};")
+    assert_fragment_type(:classref, 'c2', "class c1 : c2 {};") 
+  end
+
+  def test_struct
+    assert_fragment_type(:classdef, 'c', "struct c {};")
+    assert_fragment_type(:classdef, 'c1', "struct c1 : c2 {};")
+    assert_fragment_type(:classref, 'c2', "struct c1 : c2 {};") 
+  end
+
+  def check_scan(src)
+    LangScan::D.scan(src) {|f|
+      assert_equal(f.text, src[f.beg_byteno...f.end_byteno])
+    }
+  end
+
+  def test_sharp_in_non_initial_state
+    check_scan("struct x\n#;")
+  end
+
+  def test_fundef_returns_user_defined_type
+    assert_fragment_type(:fundef, 'foo', 'VALUE foo() {}')
+  end
+
+  def test_classdescls
+    assert_fragment_type(:classdef, 'Foo', 'class Foo { int a; }')
+    assert_fragment_type(:classdef, 'Foo', 'class Foo { int a; } var;')
+    assert_fragment_type(:classdef, 'Foo', 'abstract class Foo : A {}')
+    assert_fragment_type(:classdef, 'Foo', 'class Foo : A, B {}')
+    assert_fragment_type(:classdef, 'Foo', 'class Foo { class Inner {} }')
+    assert_fragment_type(:classdef, 'Foo', 'interface Foo {}')
+    assert_fragment_type(:classdef, 'Foo', 'struct Foo {}')
+    assert_fragment_type(:classdef, 'Foo', 'union Foo {}')
+    assert_fragment_type(:classdef, 'Foo', 'enum Foo {}')
+  end
+
+  def test_fundefs
+    assert_fragment_type(:fundef, 'foo', 'VALUE foo() {}')
+    assert_fragment_type(:fundef, 'foo', 'unittest { VALUE foo() {} }')
+    assert_fragment_type(:fundef, 'foo', 'version(HOGE) { VALUE foo() {} }')
+    assert_fragment_type(:fundef, 'foo', 'void x() { VALUE foo() {} }')
+    assert_fragment_type(:fundef, 'foo', 'void x(...) { VALUE foo() {} }')
+  end
+
+  def test_fundecls
+    assert_fragment_type(:fundecl, 'foo', 'VALUE foo();')
+    assert_fragment_type(:fundecl, 'foo', 'void x() { class C { foo(); } }')
+    assert_fragment_type(:fundecl, 'foo', 'version(HOGE) { VALUE foo(); }')
+    assert_fragment_type(:fundecl, 'foo', 'template T { VALUE foo(); }')
+    assert_fragment_type(:fundecl, 'foo', 'template T(T) { VALUE foo(); }')
+    assert_fragment_type(:fundecl, 'foo', 'template T(T,T) { VALUE foo(); }')
+    assert_fragment_type(:fundecl, 'foo', 'template T(T,T,T) { VALUE foo(); }')
+    assert_fragment_type(:fundecl, 'foo', 'template T(T:char[]) { VALUE foo(); }')
+    assert_fragment_type(:fundecl, 'foo', 'void x(int a, int b, int c, int d, ...) { class C { foo(); } }')
+  end
+
+  def test_funcalls
+    assert_fragment_type(:funcall, 'foo', 'x(foo());')
+    assert_fragment_type(:funcall, 'foo', 'void x() { foo(); }')
+    assert_fragment_type(:funcall, 'foo', 'void x() { a.foo(); }')
+    assert_fragment_type(:funcall, 'foo', 'unittest { VALUE foo(); }')
+    assert_fragment_type(:funcall, 'foo', 'void x() body { VALUE foo(); }')
+    assert_fragment_type(:funcall, 'foo', 'void x() in { VALUE foo(); }')
+    assert_fragment_type(:funcall, 'foo', 'out { VALUE foo(); }')
+    assert_fragment_type(:funcall, 'foo', 'void x() {{{{VALUE foo();}}}}')
+    assert_fragment_type(:funcall, 'foo', 'for (int i=0; i<10; i++) { VALUE foo(); }')
+    assert_fragment_type(:funcall, 'foo', 'void x() { debug(D) { VALUE foo(); } }')
+    assert_fragment_type(:funcall, 'foo', 'void x() { with(c) { VALUE foo(); } }')
+  end
+
+  def test_delegates
+    assert_fragment_type(:keyword, 'delegate', 'void delegate() fp;')
+    assert_fragment_type(:keyword, 'delegate', 'return delegate void() {}')
+    assert_fragment_type(:ident, 'type', 'return delegate type() {}')
+    assert_fragment_type(:keyword, 'function', 'void function() fp;')
+    assert_fragment_type(:keyword, 'function', 'return function type() {}')
+    assert_fragment_type(:ident, 'type', 'return function type() {}')
+  end
+
+   def test_funcables
+     assert_fragment_type(:keyword, 'new', 'c = new C();')
+     assert_fragment_type(:keyword, 'delete', 'delete p;')
+     assert_fragment_type(:keyword, 'this', 'return this;')
+     assert_fragment_type(:keyword, '~this', '~this();')
+   end
+end

data/test/langscan/d/test/test_token.rb

@@ -0,0 +1,205 @@
+require 'test/unit'
+require 'langscan/d'
+
+class TestTokenD < Test::Unit::TestCase
+  def test_d_puncts
+    t = LangScan::D::Tokenizer.new('~=')
+    type, text = t.get_token
+    assert_equal('~=', text)
+    assert_equal(:punct, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new('>>>=')
+    type, text = t.get_token
+    assert_equal('>>>=', text)
+    assert_equal(:punct, type)
+    assert_equal(nil, t.get_token)
+  end
+
+  def test_d_idents
+    t = LangScan::D::Tokenizer.new('~this')
+    type, text = t.get_token
+    assert_equal('~this', text)
+    assert_equal(:ident, type)
+    assert_equal(nil, t.get_token)
+  end
+
+  def test_d_preprocs
+    t = LangScan::D::Tokenizer.new('#line 6 "foo\bar"')
+    type, text = t.get_token
+    assert_equal('#line 6 "foo\bar"', text)
+    assert_equal(:preproc, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new('int #line 6 "foo\bar"')
+    type, text = t.get_token  # int
+    type, text = t.get_token  # space
+    type, text = t.get_token
+    assert_equal('#line 6 "foo\bar"', text)
+    assert_equal(:preproc, type)
+    assert_equal(nil, t.get_token)
+  end
+
+  def test_d_escape_integers
+    t = LangScan::D::Tokenizer.new('0XA2FlU')
+    type, text = t.get_token
+    assert_equal('0XA2FlU', text)
+    assert_equal(:integer, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new('0b100L_0127u__')
+    type, text = t.get_token
+    assert_equal('0b100L_0127u__', text)
+    assert_equal(:integer, type)
+    assert_equal(nil, t.get_token)
+  end
+
+  def test_d_escape_floatings
+    t = LangScan::D::Tokenizer.new('123_456.567_8')
+    type, text = t.get_token
+    assert_equal('123_456.567_8', text)
+    assert_equal(:floating, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new('1_2_3_4_5_6_._5e-6_fI')
+    type, text = t.get_token
+    assert_equal('1_2_3_4_5_6_._5e-6_fI', text)
+    assert_equal(:floating, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new('0x1.FFFFFFFFFFFFFp1023i')
+    type, text = t.get_token
+    assert_equal('0x1.FFFFFFFFFFFFFp1023i', text)
+    assert_equal(:floating, type)
+    assert_equal(nil, t.get_token)
+  end
+
+  def test_d_escape_strings
+    t = LangScan::D::Tokenizer.new('\?')
+    type, text = t.get_token
+    assert_equal('\?', text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new('\200')
+    type, text = t.get_token
+    assert_equal('\200', text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new('\UF0101234')
+    type, text = t.get_token
+    assert_equal('\UF0101234', text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new('\&reg;')
+    type, text = t.get_token
+    assert_equal('\&reg;', text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+  end
+
+  def test_d_escapes
+    t = LangScan::D::Tokenizer.new('"\""')
+    type, text = t.get_token
+    assert_equal('"\""', text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new('`\``')
+    type, text = t.get_token
+    assert_equal('`\`', text)
+    assert_equal(:string, type)
+    assert_not_equal(nil, t.get_token)
+  end
+
+  def test_d_characters
+    t = LangScan::D::Tokenizer.new("'a'")
+    type, text = t.get_token
+    assert_equal("'a'", text)
+    assert_equal(:character, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new("'\\''")
+    type, text = t.get_token
+    assert_equal("'\\''", text)
+    assert_equal(:character, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new("'\\x24'")
+    type, text = t.get_token
+    assert_equal("'\\x24'", text)
+    assert_equal(:character, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new("'\\&quot;'")
+    type, text = t.get_token
+    assert_equal("'\\&quot;'", text)
+    assert_equal(:character, type)
+    assert_equal(nil, t.get_token)
+  end
+
+  def test_d_string
+    t = LangScan::D::Tokenizer.new(%Q("hello"))
+    type, text = t.get_token
+    assert_equal(%Q("hello"), text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new(%Q(`hello`d))
+    type, text = t.get_token
+    assert_equal(%Q(`hello`d), text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new(%Q(x"012f"))
+    type, text = t.get_token
+    assert_equal(%Q(x"012f"), text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new(%Q(`hello`c))
+    type, text = t.get_token
+    assert_equal(%Q(`hello`c), text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new(%Q(r"hello\n"w))
+    type, text = t.get_token
+    assert_equal(%Q(r"hello\n"w), text)
+    assert_equal(:string, type)
+    assert_equal(nil, t.get_token)
+  end
+
+  def test_d_comment
+    t = LangScan::D::Tokenizer.new("/+ + +/")
+    type, text = t.get_token
+    assert_equal("/+ + +/", text)
+    assert_equal(:comment, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new("/++/")
+    type, text = t.get_token
+    assert_equal("/++/", text)
+    assert_equal(:comment, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new("/+ /++/+/")
+    type, text = t.get_token
+    assert_equal("/+ /++/+/", text)
+    assert_equal(:comment, type)
+    assert_equal(nil, t.get_token)
+
+    t = LangScan::D::Tokenizer.new("/+/++//+\n+/+//++/")
+    tok = t.get_token
+    assert_equal(:comment, tok[0])
+    assert_equal("/+/++//+\n+/+/", tok[1])
+    assert_equal(1, tok[2])
+    assert_equal(0, tok[3])
+    assert_equal(0, tok[4])
+    assert_equal(2, tok[5])
+    assert_equal(4, tok[6])
+    assert_equal(13, tok[7])
+  end
+end