langscan 1.2

data/test/langscan/lua/test/test_lua.rb

@@ -0,0 +1,69 @@
+require 'test/unit'
+require 'langscan/lua'
+
+class TestLua < Test::Unit::TestCase
+  def assert_fragments(expected, code)
+    result = []
+    LangScan::Lua.scan(code) {|f| result.push(f) }
+    fragments = []
+    result.each {|fragment|
+      type, text = fragment.to_a
+      unless type == :punct
+        fragments << [type, text]
+      end
+    }
+    assert_equal(expected, fragments)
+  end
+
+  def assert_fragment_type(type, text, src)
+    found = false
+    LangScan::Lua.scan(src) {|f|
+      if f.text == text
+        if found
+          raise "token #{text} occurred twice"
+        else
+          found = true
+          assert_equal(type, f.type, "fragment type of #{text}")
+        end
+      end
+    }
+    unless found
+      raise "token #{text} not found"
+    end
+  end
+
+  def test_comments
+    assert_fragments([[:comment, "-- c"]], "-- c")
+  end
+
+  def test_strings
+    assert_fragments([[:string, %Q("s\\"")]], %Q("s\\""))
+    assert_fragments([[:string, %Q("")]], %Q(""))
+    assert_fragments([[:string, %Q('s\\'')]], %Q('s\\''))
+    assert_fragments([[:string, %Q('')]], %Q(''))
+    assert_fragments([[:string, %Q([[s\ns]])]], %Q([[s\ns]]))
+  end
+
+  def test_integers
+    assert_fragments([[:integer, "123"]], "123")
+  end
+
+  def test_floatings
+    assert_fragments([[:floating, "123.456"]], "123.456")
+    assert_fragments([[:floating, "0.456"]], "0.456")
+    assert_fragments([[:floating, "123.456e-2"]], "123.456e-2")
+    assert_fragments([[:floating, "123.456E2"]], "123.456E2")
+  end
+
+  def test_fundefs
+    assert_fragment_type(:fundef, "foo", "function foo()")
+    assert_fragment_type(:fundef, "foo", "local function foo()")
+    assert_fragment_type(:fundef, "foo", "foo = function()")
+  end
+
+  def test_funcalls
+    assert_fragment_type(:funcall, "foo", "foo()")
+    assert_fragment_type(:funcall, "foo", "a = foo()")
+  end
+
+end

data/test/langscan/ocaml/test/test_ocaml.rb

@@ -0,0 +1,161 @@
+
+require 'test/unit'
+require 'langscan/ocaml'
+
+module LangScan::OCaml
+  class TestToken < Test::Unit::TestCase
+    def test_int
+      t = Tokenizer.new("123")
+      f = t.get_token()
+      
+      assert_equal("123", f.text)
+      assert_equal(:integer, f.type)
+    end
+    
+    def test_comment
+      t = Tokenizer.new("(* ... *)")
+      f = t.get_token()
+      assert_equal("(* ... *)", f.text)
+      assert_equal(:comment, f.type)
+      assert_equal(nil, t.get_token())
+
+      t = Tokenizer.new("(* (* (* *) *) *)")
+      f = t.get_token()      
+      assert_equal("(* (* (* *) *) *)", f.text)
+      assert_equal(:comment, f.type)
+      assert_equal(nil, t.get_token())
+
+      t = Tokenizer.new("(*\n*)")
+      f = t.get_token()
+      assert_equal("(*\n*)", f.text)
+      assert_equal(:comment, f.type)
+      assert_equal(nil, t.get_token())
+
+      t = Tokenizer.new("(* \" \" *)")
+      f = t.get_token()
+      assert_equal("(* \" \" *)", f.text)
+      assert_equal(:comment, f.type)
+      assert_equal(nil, t.get_token())
+
+      t = Tokenizer.new("(* \" *) (* \" *)")
+      f = t.get_token()
+      assert_equal("(* \" *) (* \" *)", f.text)
+      assert_equal(:comment, f.type)
+      assert_equal(nil, t.get_token())
+    
+    end
+
+    def assert_fragment_type(type, text, src)
+      found = false
+      LangScan::OCaml.scan(src) {|f|
+        if f.text == text
+          if found
+            raise "token #{text} occurred twice"
+          else
+            found = true
+            assert_equal(type, f.type, "fragment type of #{text}")
+          end
+        end
+      }
+      unless found
+        raise "token #{text} not found"
+      end
+    end
+
+    def test_fundefs
+      assert_fragment_type(:ident, "foo", "let foo = bar")
+      assert_fragment_type(:fundef, "foo", "let foo bar = bar")
+      assert_fragment_type(:fundef, "foo", "let rec foo bar = bar")
+      assert_fragment_type(:funcall, "baz", "let foo bar = baz bar")
+    end
+
+    def test_methdefs
+      assert_fragment_type(:fundef, "foo", "method foo = 0")
+      assert_fragment_type(:funcall, "foo", "method f b = foo b")
+      assert_fragment_type(:ident, "foo", "method f foo c = a")
+
+      assert_fragment_type(:fundef, "foo", "method private foo = 0")
+      assert_fragment_type(:funcall, "foo", "method virtual f b = foo b")
+      assert_fragment_type(:ident, "foo", "method private virtual f foo c = a")
+    end
+
+    def test_anon_fun
+      assert_fragment_type(:ident, "foo", "fun foo bar baz = baz")
+      assert_fragment_type(:ident, "bar", "fun foo bar baz = baz")
+    end
+
+    def test_funcalls
+      assert_fragment_type(:funcall, "foo", "foo bar")
+      assert_fragment_type(:ident, "foo", "a:foo b:bar")
+      assert_fragment_type(:ident, "foo", "| foo bar")
+      assert_fragment_type(:ident, "foo", "with foo bar")
+      assert_fragment_type(:ident, "foo", "* foo bar")   # not real?
+      assert_fragment_type(:ident, "foo", "'foo bar")
+      assert_fragment_type(:funcall, "foo", "foo 1")
+      assert_fragment_type(:funcall, "foo", "foo 1.1")
+      assert_fragment_type(:funcall, "foo", "foo \"str\"")
+      assert_fragment_type(:funcall, "foo", "foo ()")
+      assert_fragment_type(:ident, "foo", "foo fun")   # not real?
+      assert_fragment_type(:ident, "foo", "foo -1")
+      assert_fragment_type(:funcall, "foo", "foo (-1)")
+      assert_fragment_type(:funcall, "foo", "foo [ 1 ]")
+      assert_fragment_type(:funcall, "foo", "foo [| 1 |]")
+      assert_fragment_type(:ident, "foo", "foo list")
+      assert_fragment_type(:ident, "foo", "?foo bar")
+      assert_fragment_type(:ident, "foo", "~foo bar")
+    end
+
+    def test_after_funcalls
+      assert_fragment_type(:funcall, "foo", "f (foo bar)")
+      assert_fragment_type(:ident, "foo", "f [|1|] foo bar")
+      assert_fragment_type(:ident, "foo", "f a.b foo bar")
+      assert_fragment_type(:ident, "foo", "f a#b foo bar")
+      assert_fragment_type(:ident, "foo", "f a.b c.foo bar")
+      assert_fragment_type(:ident, "foo", "f a#b c#foo bar")
+    end
+
+    def test_methcalls
+      assert_fragment_type(:funcall, "foo", "o#foo 1")
+      assert_fragment_type(:funcall, "foo", "o#foo;")
+    end
+    
+    def test_fundecls
+      assert_fragment_type(:ident, "foo", "val foo : int")
+      assert_fragment_type(:fundecl, "foo", "val foo : int -> int")
+      assert_fragment_type(:fundecl, "foo", "val mutable foo : int -> int")
+      assert_fragment_type(:fundecl, "foo", "external foo : int -> int")
+    end
+
+    def test_methdecls
+      assert_fragment_type(:fundecl, "foo", "method foo : int -> int")
+      assert_fragment_type(:fundecl, "foo", "method foo : int")
+      assert_fragment_type(:ident, "foo", "method f : (int * int) foo b")
+      assert_fragment_type(:ident, "foo", "method f : a:foo b")
+
+      assert_fragment_type(:fundecl, "foo", "method private foo : int")
+      assert_fragment_type(:fundecl, "foo", "method virtual foo : int")
+      assert_fragment_type(:fundecl, "foo", "method private virtual foo : int")
+    end
+
+    def test_skiptypes
+      assert_fragment_type(:ident, "foo", "of foo bar")
+      assert_fragment_type(:ident, "foo", ":> foo bar")
+#      assert_fragment_type(:ident, "foo", "| f.foo bar")
+    end
+
+    def test_class
+      assert_fragment_type(:ident, "foo", "class foo bar")
+    end
+
+    def test_skip_comments
+      assert_fragment_type(:fundecl, "foo", "method (**) foo (**) : (**) int")
+      assert_fragment_type(:funcall, "foo", "foo (**) [| (**) 1 (**) |]")
+      assert_fragment_type(:fundef, "foo",
+                           "let (**) rec (**) foo (**) bar (**) = (**) bar")
+    end
+
+  end if system("ocaml -version")
+
+end
+
+

data/test/langscan/php/test/test_scan.rb

@@ -0,0 +1,138 @@
+require 'test/unit'
+require 'langscan/php'
+
+class TestScan < Test::Unit::TestCase
+  def assert_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      yielded = true
+      yield(*block_args)
+    }
+    assert(yielded, "block not yielded")
+  end
+  
+  def assert_not_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      assert(false, "block yielded")
+    }
+  end
+  
+  def assert_yield_any(recv, meth, *args)
+    success = false
+    recv.__send__(meth, *args) {|*block_args|
+      if yield(*block_args)
+        assert(true)
+        success = true
+        break
+      end
+    }
+    assert(false, "no expected yields") unless success
+  end
+  
+  def assert_yield_all(recv, meth, *args)
+    recv.__send__(meth, *args) {|*block_args|
+      if !yield(*block_args)
+        assert(false, "unexpected yields")
+        return
+      end
+    }
+    assert(true)
+  end
+  
+  def test_classdef
+    assert_yield_any(LangScan::PHP, :scan, "<?php class foo{} ?>") {|f|
+      f.type == :classdef && f.text == 'foo'
+    }
+  end
+  
+  def test_fundef
+    assert_yield_any(LangScan::PHP, :scan, "<?php function foo1() {} ?>") {|f|
+      f.type == :fundef && f.text == 'foo1'
+    }
+  end
+
+  def test_funcall
+    assert_yield_any(LangScan::PHP, :scan, "<?php hoge(); ?>") {|f|
+      f.type == :funcall && f.text == 'hoge'
+    }
+    assert_yield_all(LangScan::PHP, :scan, "<?php if(true); ?>") {|f|
+      f.type != :funcall
+    }
+  end
+
+  def test_string
+    [
+      "'hoge'",
+      '"hoge"',
+      '"hoge\nfuga"'
+    ].each do |src|
+      assert_yield_any(LangScan::PHP, :scan, "<?php #{src} ?>") {|f|
+        f.type == :string
+      }
+    end
+  end
+
+  def test_comment
+    [
+      '# comment',
+      "# comment\n",
+      "fuga(); # comment\nhoga();",
+      "fuga(); # comment",
+    ].each do |src|
+      assert_yield_any(LangScan::PHP, :scan, "<?php #{src} ?>") {|f|
+        f.type == :comment
+      }
+      assert_yield_any(LangScan::PHP, :scan, "<?php #{src} ?>") {|f|
+        f.type == :ident and f.text == "?>"
+      }
+    end
+  end
+
+  def test_space
+    [
+      '3 + 2',
+      "hoge( fuga)"
+    ].each do |src|
+      assert_yield_any(LangScan::PHP, :scan, "<?php #{src} ?>") {|f|
+        f.type == :space
+      }
+    end
+  end
+
+  def test_heredoc
+    [
+      "echo <<<EOS\r\nfoo\nEOS;\n",
+      "echo <<<EOS\r\nE\rEOS;\n",
+      "echo <<<EOS\rHOGE;\nEOS\n",
+      "echo <<<  EODA\nHGE;\rEODA\n",
+      "echo <<<\tE\nF\nE\r",
+      "echo <<<EOS\r\nm\rEOS\n",
+      "echo <<<EO#{'S'*1000}\r\nEOS;\nEO#{'S'*1000}\n",
+    ].each do |src|
+      pat = "<?php #{src} ?>"
+      assert_yield_any(LangScan::PHP, :scan, pat) {|f|
+        f.type == :string
+      }
+      assert_yield_any(LangScan::PHP, :scan, pat) {|f|
+        f.type == :ident and f.text == "?>"
+      }
+    end
+  end
+
+  def test_scriptregion
+    [
+      '<?php "hoge" ?>',
+      '<%= "hoge" %>',
+      '<% "hoge" %>',
+      '<script language="php"> "hoge" </script>',
+      '<? "hoge" ?>',
+      '<?= "hoge" ?>',
+    ].each do |src|
+      assert_yield_any(LangScan::PHP, :scan, src) {|f|
+        f.type == :string and f.text == '"hoge"'
+      }
+    end
+  end
+end
+

data/test/langscan/python/test/test_scan.rb

@@ -0,0 +1,105 @@
+require 'test/unit'
+require 'langscan/python'
+
+class TestScan < Test::Unit::TestCase
+  def assert_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      yielded = true
+      yield(*block_args)
+    }
+    assert(yielded, "block not yielded")
+  end
+  
+  def assert_not_yield(recv, meth, *args)
+    yielded = false
+    recv.__send__(meth, *args) {|*block_args|
+      assert(false, "block yielded")
+    }
+  end
+  
+  def assert_yield_any(recv, meth, *args)
+    success = false
+    recv.__send__(meth, *args) {|*block_args|
+      if yield(*block_args)
+        assert(true)
+        success = true
+        break
+      end
+    }
+    assert(false, "no expected yields") unless success
+  end
+  
+  def assert_yield_all(recv, meth, *args)
+    recv.__send__(meth, *args) {|*block_args|
+      if !yield(*block_args)
+        assert(false, "unexpected yields")
+        return
+      end
+    }
+    assert(true)
+  end
+  
+  def test_classdef
+    assert_yield_any(LangScan::Python, :scan, "class Bar:\n\tpass") {|f|
+      f.type == :classdef && f.text == 'Bar'
+    }
+  end
+  
+  def test_fundef
+    assert_yield_any(LangScan::Python, :scan, "def Foo:\n\tpass") {|f|
+      f.type == :fundef && f.text == 'Foo'
+    }
+  end
+  
+  def test_funcall
+    assert_yield_any(LangScan::Python, :scan, "hoge()") {|f|
+      f.type == :funcall && f.text == 'hoge'
+    }
+  end
+  
+  def test_not_funcall
+    assert_yield_any(LangScan::Python, :scan, "if (1)") {|f|
+      f.type == :keyword && f.text == 'if'
+    }
+  end
+
+  def test_string
+    ['r"""'+"\n"+'hoge'+"\n"+'"""',
+     'r"""hoge"""',
+     "r'''\nhoge\n'''",
+     "r'''hoge'''",
+     'r"hoge"',
+     "r'hoge'",
+     "'hoge'",
+     '"hoge"',
+     '"hoge\nfuga"'].each do |src|
+      assert_yield_any(LangScan::Python, :scan, src) {|f|
+        f.type == :string
+      }
+    end
+  end
+  
+  def test_comment
+    [
+      '# comment',
+      "hoge.fuga() # comment"
+    ].each do |src|
+      assert_yield_any(LangScan::Python, :scan, src) {|f|
+        f.type == :comment
+      }
+    end
+  end
+  
+  def test_space
+    [
+      '3 + 2',
+      "hoge( fuga)"
+    ].each do |src|
+      assert_yield_any(LangScan::Python, :scan, src) {|f|
+        f.type == :space
+      }
+    end
+  end
+end
+