langscan 1.2

data/test/langscan/eiffel/test/test_eiffel.rb

@@ -0,0 +1,95 @@
+require 'test/unit'
+require 'langscan/eiffel'
+
+class TestEiffel < Test::Unit::TestCase
+  def assert_fragments(expected, code)
+    result = []
+    LangScan::Eiffel.scan(code) {|f| result.push(f) }
+    fragments = []
+    result.each {|fragment|
+      type, text = fragment.to_a
+      unless type == :punct
+        fragments << [type, text]
+      end
+    }
+    assert_equal(expected, fragments)
+  end
+
+  def assert_fragment_type(type, text, src)
+    found = false
+    LangScan::Eiffel.scan(src) {|f|
+      if f.text == text
+        if found
+          raise "token #{text} occurred twice"
+        else
+          found = true
+          assert_equal(type, f.type, "fragment type of #{text}")
+        end
+      end
+    }
+    unless found
+      raise "token #{text} not found"
+    end
+  end
+
+  def test_comments
+    assert_fragments([[:comment, "-- c"]], "-- c")
+  end
+
+  def test_strings
+    assert_fragments([[:string, %Q("s%"")]], %Q("s%""))
+    assert_fragments([[:string, %Q("")]], %Q(""))
+    assert_fragments([[:character, %Q('%'')]], %Q('%''))
+    assert_fragments([[:character, %Q('c')]], %Q('c'))
+    assert_fragments([[:string, %Q("s%\ns")]], %Q("s%\ns"))
+  end
+
+  def test_integers
+    assert_fragments([[:integer, "123"]], "123")
+    assert_fragments([[:integer, "1_234"]], "1_234")
+  end
+
+  def test_floatings
+    assert_fragments([[:floating, "1_234.456"]], "1_234.456")
+    assert_fragments([[:floating, ".456"]], ".456")
+    assert_fragments([[:floating, "123."]], "123.")
+    assert_fragments([[:floating, "123.456e-2"]], "123.456e-2")
+    assert_fragments([[:floating, "123.456E2"]], "123.456E2")
+    assert_fragments([[:floating, "123.456E+2"]], "123.456E+2")
+    assert_fragments([[:floating, ".E+2"]], ".E+2")
+    assert_fragments([], ".")
+  end
+
+  def test_keywords
+    assert_fragment_type(:keyword, 'feature', 'feature')
+    assert_fragment_type(:keyword, 'fEAtUrE', 'fEAtUrE')
+  end
+
+  def test_fundefs
+    assert_fragment_type(:fundef, 'foo', 'foo() is do')
+    assert_fragment_type(:fundef, 'foo', 'foo(p : INTEGER) is')
+    assert_fragment_type(:fundef, 'foo', 'foo(p : INTEGER) : INTEGER is')
+    assert_fragment_type(:fundef, 'foo', 'foo is do')
+    assert_fragment_type(:fundef, 'foo', 'foo: INTEGER is local')
+    assert_fragment_type(:fundef, 'foo', "foo: INTEGER is --comment\nexternal")
+
+    assert_fragment_type(:ident, 'foo', 'foo is REAL')
+    assert_fragment_type(:ident, 'foo', 'f (p : INTEGER) : foo is once')
+  end
+
+  def test_funcalls
+    assert_fragment_type(:funcall, 'foo', 'foo();')
+    assert_fragment_type(:funcall, 'foo', 'a.foo(p);')
+  end
+
+  def test_renames
+    assert_fragment_type(:fundef, 'foo', 'rename f as foo')
+    assert_fragment_type(:ident, 'foo', 'rename foo as f')
+    assert_fragment_type(:fundef, 'foo', 'rename b as bar, f as foo')
+  end
+
+  def test_classdefs
+    assert_fragment_type(:classdef, 'foo', 'class foo')
+  end
+
+end

data/test/langscan/elisp/test/test_elisp.rb

@@ -0,0 +1,177 @@
+$LOAD_PATH << "../../.." unless $LOAD_PATH.include?("../../..")
+require 'test/unit'
+require 'langscan/elisp'
+
+class TestEmacsLisp < Test::Unit::TestCase
+  def assert_yield_any(recv, meth, *args)
+    success = false
+    recv.__send__(meth, *args) {|*block_args|
+      if yield(*block_args)
+        assert(true)
+        success = true
+        break
+      end
+    }
+    assert(false, "no expected yields") unless success
+  end
+
+  def assert_yield_all(recv, meth, *args)
+    recv.__send__(meth, *args) {|*block_args|
+      if !yield(*block_args)
+        assert(false, "unexpected yields")
+        return
+      end
+    }
+    assert(true)
+  end
+
+  def test_defun
+    [
+      {:str => "(defun foo () ())", :text => "foo"},
+      {:str => "(defun foo (bar) ())", :text => "foo"},
+      {:str => "(defun foo (bar) baz)", :text => "foo"},
+      {:str => "(defun foo= (a) (= a a))", :text => "foo="}
+    ].each {|src|
+      assert_yield_any(LangScan::EmacsLisp, :scan, src[:str]) {|f|
+        f.type == :fundef && f.text == src[:text]
+      }
+    }
+  end
+
+  def test_funcall
+    [
+      {:str => "(foo)", :text => "foo"},
+      {:str => "(foo bar)", :text => "foo"},
+      {:str => "(let ((a 1) (b 2)) (+ a b))", :text => "+"}
+    ].each {|src|
+      assert_yield_any(LangScan::EmacsLisp, :scan, src[:str]) {|f|
+        f.type == :funcall && f.text == src[:text]
+      }
+    }
+  end
+
+  def test_defvar
+    assert_yield_any(LangScan::EmacsLisp, :scan, "(defvar a 't)") {|f|
+      f.type == :keyword && f.text == 'defvar'
+    }
+  end
+
+  def test_setq
+    assert_yield_any(LangScan::EmacsLisp, :scan, "(setq a 1)") {|f|
+      f.type == :keyword && f.text == 'setq'
+    }
+  end
+
+  def test_ident
+    %w(foo foo= foo? char-to-string 1+ \+1 +-*/_~!@$%^&=:<>{}
+    ).each {|src|
+      assert_yield_any(LangScan::EmacsLisp, :scan, src) {|f|
+        f.type == :ident
+      }
+    }
+  end
+
+  def test_quote
+    [
+      "'(a b c)",
+      "`(a b c)",
+      "(quote (a b c))",
+      "(backquote (a b c))",
+    ].each { |src|
+      assert_yield_all(LangScan::EmacsLisp, :scan, src) {|f|
+        f.type != :funcall
+      }
+    }
+  end
+
+  def test_character
+    [
+      '?a',
+      '? ',
+      '?\C-\\',
+      '?\C-\M-m',
+      '?\101',
+      '?\)',
+      '?\^Ij',
+      '?\H-\M-\A-x'
+    ].each { |src|
+      assert_yield_any(LangScan::EmacsLisp, :scan, src) {|f|
+        f.type == :character
+      }
+    }
+  end
+
+  def test_string
+    [
+      '"foo"',
+      "\"foo\nbar\"",
+      "\"foo\\\nbar\"",
+      '"foo\"bar"',
+      '"foo\\\\"',
+    ].each { |src|
+      assert_yield_any(LangScan::EmacsLisp, :scan, src) {|f|
+        f.type == :string
+      }
+    }
+  end
+
+  def test_number
+    %w( 1 -1 1050 10. +1 ).each { |src|
+      assert_yield_any(LangScan::EmacsLisp, :scan, src) {|f|
+        f.type == :integer
+      }
+    }
+    %w( 1.0 -.1 4e10 10.3e4 +3.32e-10 ).each { |src|
+      assert_yield_any(LangScan::EmacsLisp, :scan, src) {|f|
+        f.type == :floating
+      }
+    }
+  end
+
+  def test_comment
+    [
+      '; comment',
+      "(foo bar) ; comment"
+    ].each { |src|
+      assert_yield_any(LangScan::EmacsLisp, :scan, src) {|f|
+        f.type == :comment
+      }
+    }
+  end
+
+
+  def assert_fragment_type(type, text, src)
+    found = false
+    LangScan::EmacsLisp.scan(src) {|f|
+      if f.text == text
+        if found
+          raise "token #{text} occurred twice"
+        else
+          found = true
+          assert_equal(type, f.type, "fragment type of #{text}")
+        end
+      end
+    }
+    unless found
+      raise "token #{text} not found"
+    end
+  end
+
+  def test_nest_in_quote
+    assert_fragment_type(:ident, 'a', "'(a (b))")
+    assert_fragment_type(:ident, 'a', "(quote (a (b)))")
+    assert_fragment_type(:ident, 'b', "'(a (b))")
+    assert_fragment_type(:ident, 'b', "`(a (b))")
+    assert_fragment_type(:ident, 'b', "(backquote (a (b)))")
+    assert_fragment_type(:funcall, 'b', "`(a ,(b))")
+    assert_fragment_type(:funcall, 'b', "`(a ,@(b))")
+  end
+
+  def test_let_body
+    assert_fragment_type(:funcall, 'y', "(let ((x 1)) ((y)))")
+    assert_fragment_type(:funcall, 'y', "(let ((x 1) (z 2)) ((y)))")
+    assert_fragment_type(:funcall, 'y', "(let ((x 1) (z 2)) (y))")
+    assert_fragment_type(:ident, 'z', "(let ((x 1) (z 2)) ((y)))")
+  end
+
+end

data/test/langscan/io/test/test_io.rb

@@ -0,0 +1,79 @@
+require 'test/unit'
+require 'langscan/io'
+
+class TestIo < Test::Unit::TestCase
+  def assert_fragments(expected, code)
+    result = []
+    LangScan::Io.scan(code) {|f| result.push(f) }
+    fragments = []
+    result.each {|fragment|
+      type, text = fragment.to_a
+      unless type == :punct
+        fragments << [type, text]
+      end
+    }
+    assert_equal(expected, fragments)
+  end
+
+  def assert_punct(expected, code)
+    num = 0
+    LangScan::Io.scan(code) {|f|
+      assert_equal(:punct, f.type)
+      assert_equal(expected, f.text)
+      num += 1
+    }
+    assert_equal(1, num)
+  end
+
+  def test_comments
+    assert_fragments([[:comment, "/*c*/"]], "/*c*/")
+    assert_fragments([[:comment, "/*/c\n*/"]], "/*/c\n*/")
+    assert_fragments([[:comment, "#c"]], "#c")
+    assert_fragments([[:comment, "//c"]], "//c")
+  end
+
+  def test_strings
+    assert_fragments([[:string, %Q("s")]], %Q("s"))
+    assert_fragments([[:string, %Q("""s\ns""")]], %Q("""s\ns"""))
+    assert_fragments([[:string, %Q("")]], %Q(""))
+  end
+
+  def test_integers
+    assert_fragments([[:integer, "123"]], "123")
+    assert_fragments([[:integer, "0x0"]], "0x0")
+    assert_fragments([[:integer, "0x0F"]], "0x0F")
+    assert_fragments([[:integer, "0XeE"]], "0XeE")
+  end
+
+  def test_floatings
+    assert_fragments([[:floating, "123.456"]], "123.456")
+    assert_fragments([[:floating, "0.456"]], "0.456")
+    assert_fragments([[:floating, ".456"]], ".456")
+    assert_fragments([[:floating, "123e-2"]], "123e-2")
+    assert_fragments([[:floating, "123e2"]], "123e2")
+    assert_fragments([[:floating, "123.456e-2"]], "123.456e-2")
+    assert_fragments([[:floating, "123.456e2"]], "123.456e2")
+  end
+
+  def test_fundefs
+    assert_fragments([[:fundef, "s"], [:funcall, "f"]], "s := f")
+    assert_fragments([[:fundef, "s"], [:funcall, "f"]], "s = f")
+  end
+
+  def test_funcalls
+    assert_fragments([[:funcall, "f"]], "f")
+    assert_fragments([[:funcall, "a"]], "@a")
+    assert_fragments([[:funcall, "a"]], "@@a")
+  end
+
+  def test_punct
+    assert_punct('@@', '@@')
+    assert_punct(':=', ':=')
+    assert_punct(';', ';')
+    assert_punct('(', '(')
+    assert_punct('{', '{')
+    assert_punct(']', ']')
+    assert_punct(',', ',')
+  end
+
+end

data/test/langscan/java/test/test_java.rb

@@ -0,0 +1,74 @@
+require 'test/unit'
+require 'langscan/java'
+
+class TestJava < Test::Unit::TestCase
+  def test_duplicated
+    fs = {}
+    LangScan::Java.scan("a()") {|f|
+      assert(!fs[f.beg_byteno], "duplicated fragment: #{f.inspect}")
+      fs[f.beg_byteno] = f
+    }
+  end
+
+  def test_fundef
+    fragments = scan("int foo() {}")
+    fundef = fragments.find_all {|f| f.type == :fundef}
+    assert_equal(1, fundef.size, fragments.join("\n"))
+    assert_equal("foo", fundef[0].text)
+    assert_equal(4, fundef[0].beg_byteno)
+    assert_equal(1, fundef[0].beg_lineno)
+  end
+
+  def test_fundef
+    fragments = scan("int\nfoo()\n{}")
+    fundef = fragments.find_all {|f| f.type == :fundef}
+    assert_equal(1, fundef.size, fragments.join("\n"))
+    assert_equal("foo", fundef[0].text)
+    assert_equal(4, fundef[0].beg_byteno)
+    assert_equal(2, fundef[0].beg_lineno)
+  end
+
+  def test_fundef_arg
+    fragments = scan("String getName(Object obj) {}")
+    fundef = fragments.find_all {|f| f.type == :fundef}
+    assert_equal(1, fundef.size, fragments.join("\n"))
+    assert_equal("getName", fundef[0].text)
+  end
+
+  def test_fundef_comment
+    fragments = scan("String getName(Object obj, /* (ok) */) {}")
+    fundef = fragments.find_all {|f| f.type == :fundef}
+    assert_equal(1, fundef.size, fragments.join("\n"))
+    assert_equal("getName", fundef[0].text)
+  end
+
+  def test_fundef_comment2
+    fragments = scan("public static final String getName(Object obj, // (ok)\n) {}")
+    fundef = fragments.find_all {|f| f.type == :fundef}
+    assert_equal(1, fundef.size, fragments.join("\n"))
+    assert_equal("getName", fundef[0].text)
+    assert_equal(27, fundef[0].beg_byteno)
+    assert_equal(1, fundef[0].beg_lineno)
+  end
+
+  def test_fundef_throws
+    fragments = scan("int foo() throws IOException {}")
+    fundef = fragments.find_all {|f| f.type == :fundef}
+    assert_equal(1, fundef.size, fragments.join("\n"))
+    assert_equal("foo", fundef[0].text)
+  end
+
+  def test_fundef_throws2
+    fragments = scan("int foo() throws IOException, GonzuiException {}")
+    fundef = fragments.find_all {|f| f.type == :fundef}
+    assert_equal(1, fundef.size, fragments.join("\n"))
+    assert_equal("foo", fundef[0].text)
+  end
+
+  def scan(src)
+    r = []
+    LangScan::Java.scan(src) {|f| r << f }
+    return r
+  end
+  private :scan
+end

data/test/langscan/javascript/test/test_javascript.rb

@@ -0,0 +1,39 @@
+$LOAD_PATH << "../../.." unless $LOAD_PATH.include?("../../..")
+require 'test/unit'
+require 'langscan/javascript'
+
+class TestJavaScript < Test::Unit::TestCase
+  def assert_yield_any(recv, meth, *args)
+    success = false
+    recv.__send__(meth, *args) {|*block_args|
+      if yield(*block_args)
+        assert(true)
+        success = true
+        break
+      end
+    }
+    assert(false, "no expected yields") unless success
+  end
+
+  def assert_yield_all(recv, meth, *args)
+    recv.__send__(meth, *args) {|*block_args|
+      if !yield(*block_args)
+        assert(false, "unexpected yields")
+        return
+      end
+    }
+    assert(true)
+  end
+
+  def test_funcall
+    assert_yield_any(LangScan::JavaScript, :scan, "a()") {|f|
+      f.type == :funcall && f.text == 'a'
+    }
+  end
+
+  def test_fundef
+    assert_yield_any(LangScan::JavaScript, :scan, "a(){}") {|f|
+      f.type == :fundef && f.text == 'a'
+    }
+  end
+end