langscan 1.2

data/lib/langscan/c.rb

@@ -0,0 +1,144 @@
+#
+# c.rb - a C module of LangScan
+#
+# Copyright (C) 2004-2005 Akira Tanaka <akr@m17n.org> 
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'langscan/c/c'
+require 'langscan/_common'
+require 'langscan/_pairmatcher'
+
+module LangScan
+  module C
+    module_function
+    def name
+      "C/C++"
+    end
+
+    def abbrev
+      "c"
+    end
+
+    def extnames
+      [".c", ".h", ".cc", ".cpp"]
+    end
+
+    # LangScan::C.scan iterates over C program.
+    # It yields for each Fragment.
+    def scan(input, &block)
+      sorter = PairMatcher.fragmentsorter(block)
+      scan_unsorted(input, &sorter)
+    end
+
+    def scan_unsorted(input, &block)
+      pm = LangScan::PairMatcher.new(3,2,2,2)
+      pm.define_intertoken_fragment :space, nil
+      pm.define_intertoken_fragment :comment, nil
+      pm.define_pair :paren, :punct, "(", :punct, ")"
+      pm.define_pair :brace, :punct, "{", :punct, "}"
+      pm.define_pair :bracket, :punct, "[", :punct, "]"
+      pm.define_pair :preproc, :preproc_beg, "#", :preproc_end, "\n"
+      pm.parse(LangScan::C::Tokenizer.new(input), lambda {|f|
+        if f.type == :ident
+          f.type = IdentType[f.text]
+        end
+        yield f
+      }) {|pair|
+        if pair.pair_type == :paren
+          if 1 <= pair.before_open_length
+            fun = pair.around_open(-1)
+            if fun.type == :ident && IdentType[fun.text] == :ident
+              # ident(...)
+              if (outer = pair.outer) && pair.outmost.pair_type == :paren
+                # type ident(type (*arg)());
+              elsif outer &&
+                 outer.pair_type == :preproc &&
+                 2 <= outer.after_open_length &&
+                 outer.around_open(1).type == :ident && /\Adefine\z/ =~ outer.around_open(1).text &&
+                 outer.around_open(2) == pair.around_open(-1)
+                # #define ident(...)
+                # #define ident (...)
+                if pair.around_open(-1).end_byteno == pair.open_token.beg_byteno
+                  # #define ident(...)
+                  fun.type = :fundef
+                end
+              elsif !outer ||
+                    (!outer.outer && # extern "C" { ... }
+                     outer.pair_type == :brace &&
+                     2 <= outer.before_open_length &&
+                     outer.around_open(-2).type == :ident && /\Aextern\z/ =~ outer.around_open(-2).text &&
+                     outer.around_open(-1).type == :string && /\A"C"\z/ =~ outer.around_open(-1).text)
+                if 2 <= pair.before_open_length &&
+                   pair.around_open(1).type == :punct && pair.around_open(1).text == '(' &&
+                   pair.around_close(-1).type == :punct && pair.around_close(-1).text == ')' &&
+                   pair.around_open(-2).type == :ident && IdentType[pair.around_open(-2).text] == :ident
+                  # ident ident((...))
+                  pair.around_open(-2).type = :fundecl
+                elsif 1 <= pair.after_close_length &&
+                      pair.around_close(1).type == :punct && /\A;\z/ =~ pair.around_close(1).text
+                  # ident(...);
+                  fun.type = :fundecl
+                elsif 1 <= pair.after_close_length &&
+                      ((pair.around_close(1).type == :punct && /\A\{\z/ =~ pair.around_close(1).text) || # }
+                       (pair.around_close(1).type == :ident))
+                  # name(...) { ... }
+                  # name(...) int arg; { ... }
+                  # name(...) struct tag *arg; { ... }
+                  # name(...) typedefed_type arg; { ... }
+                  fun.type = :fundef
+                else
+                  fun.type = :funcall
+                end
+              else
+                if /\Adefined\z/ =~ fun.text &&
+                   (outer = pair.outer) &&
+                   !outer.outer &&
+                   outer.pair_type == :preproc &&
+                   1 <= outer.after_open_length &&
+                   /\Aif\z/ =~ outer.around_open(1).text
+                  # #if ... defined(...)
+                else
+                  fun.type = :funcall
+                end
+              end
+            end
+          end
+        end
+      }
+    end
+
+    Keywords = %w(
+      auto break case char const continue default do
+      double else enum extern float for goto if int
+      long register return short signed sizeof static
+      struct switch typedef union unsigned void volatile
+      while
+    )
+    KeywordsHash = {}
+    Keywords.each {|k| KeywordsHash[k] = k }
+
+    Types = %w(char double float int long short void)
+    TypesHash = {}
+    Types.each {|k| TypesHash[k] = k }
+
+    IdentType = Hash.new(:ident)
+    Keywords.each {|k| IdentType[k] = :keyword }
+    Types.each {|k| IdentType[k] = :type }
+
+    # for debug
+    def C.each_fragment(input)
+      tokenizer = LangScan::C::Tokenizer.new(input)
+      while t = tokenizer.get_token
+        yield t
+      end
+    end
+
+    LangScan.register(self)
+  end
+end
+

data/lib/langscan/csharp.rb

@@ -0,0 +1,101 @@
+#
+# csharp.rb - a C# module of LangScan
+#
+# Copyright (C) 2005 Kenichi Ishibashi <bashi at dream.ie.ariake-nct.ac.jp>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'langscan/csharp/csharp'
+require 'langscan/_common'
+require 'langscan/_pairmatcher'
+
+module LangScan
+  module Csharp
+    module_function
+    def name
+      "C#"
+    end
+
+    def abbrev
+      "csharp"
+    end
+
+    def extnames
+      [".cs"]
+    end
+
+    # LangScan::Csharp.scan iterates over Csharp program.
+    # It yields for each Fragment.
+    def scan(input, &block)
+      sorter = PairMatcher.fragmentsorter(block)
+      scan_unsorted(input, &sorter)
+    end
+
+    def scan_unsorted(input, &block)
+      pm = LangScan::PairMatcher.new(1,0,0,1)
+      pm.define_intertoken_fragment :space, nil
+      pm.define_intertoken_fragment :comment, nil
+      pm.define_pair :paren, :punct, "(", :punct, ")"
+      pm.parse(LangScan::Csharp::Tokenizer.new(input), lambda {|f|
+        if f.type == :ident
+          f.type = IdentType[f.text]
+        end
+        if f.type == :delegate
+          f.type = :ident
+        end
+        yield f
+      }) {|pair|
+        if 1 <= pair.before_open_length &&
+           pair.around_open(-1).type == :ident &&
+           IdentType[pair.around_open(-1).text] == :ident
+          before_open_token = pair.around_open(-1)
+          if !KeywordsHash[before_open_token.text]
+            if !(outer = pair.outer) || !outer.outer
+              if 1 <= pair.after_close_length &&
+                 (pair.around_close(1).type == :punct &&
+                  pair.around_close(1).text == '{' ||
+                  pair.around_close(1).text == ':')
+                before_open_token.type = :fundef
+              else
+                before_open_token.type = :funcall
+              end
+            else
+              before_open_token.type = :funcall
+            end
+          end
+        end
+      }
+    end
+
+    Keywords = %w(
+      abstract as base bool break byte case catch char checked class const
+      continue decimal default delegate do double else enum event explicit
+      extern false finally fixed float for foreach goto if implicit in int
+      interface internal is lock long namespace new null object operator
+      out override params private protected public readonly ref return sbyte
+      sealed short sizeof stackalloc static string struct switch this throw
+      true try typeof uint ulong unchecked unsafe ushort using virtual void
+      volatile while
+    )
+    KeywordsHash = {}
+    Keywords.each {|k| KeywordsHash[k] = k }
+
+    Types = %w(
+      bool byte char double decimal float int long sbyte short uint ulong
+      ushort void
+    )
+    TypesHash = {}
+    Types.each {|k| TypesHash[k] = k }
+
+    IdentType = Hash.new(:ident)
+    Keywords.each {|k| IdentType[k] = :keyword }
+    Types.each {|k| IdentType[k] = :type }
+
+    LangScan.register(self)
+  end
+end
+

data/lib/langscan/css.rb

@@ -0,0 +1,109 @@
+#
+# css.rb - a CSS module of LangScan
+#
+# Copyright (C) 2005 Kouichirou Eto <2005 at eto.com>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'langscan/_easyscanner'
+
+module LangScan
+  module CSS
+    module_function
+    def name
+      "CSS"
+    end
+
+    def abbrev
+      "css"
+    end
+
+    def extnames
+      [".css"]
+    end
+
+    Pattern = [
+      [:comment, "/\\*", "\\*/"],
+      [:string, "\"", "[^\\\\]\""],
+      [:string, "\\(", "[^\\\\]\\)"],
+      [:keyword, "\\!\s*important"],
+#     [:ident, "[-@\\.\\#\\>\\w]+"],
+      [:ident, "[-@\\w]+"],
+      [:integer, "\\d[\\.\\w\\d%]+"],
+      [:punct, "\\."],
+      [:punct, "\\#"],
+      [:punct, "\\{"],
+      [:punct, "\\}"],
+      [:punct, "\\:"],
+      [:punct, "\\;"],
+    ]
+
+    Types = []
+
+    Keywords = %w(
+      url
+      @import
+      important
+    )
+
+    def goback(new_tokens)
+      for i in 0...new_tokens.length
+	past_token = new_tokens[new_tokens.length-1-i] # take it from the last
+	if past_token
+	  if past_token.type == :ident || past_token.type == :keyword
+	    past_token.type = :fundef
+	  end
+
+	  if past_token.type == :punct &&
+	      (past_token.text == "}" || past_token.text == ";")
+	    break
+	  end
+	end
+      end
+    end
+
+    def parse_token(t, new_tokens)
+      last_token = new_tokens.last
+      return if last_token.nil?
+
+      return unless t.type == :punct and last_token.type == :ident
+
+      if t.text == ':'
+	last_token.type = :keyword
+	return
+      end
+
+      if t.text == '{'
+	goback(new_tokens)
+	return
+      end
+    end
+
+    # LangScan::CSS.scan iterates over CSS file.
+    # It yields for each Fragment.
+    def scan(input, &block)
+      scanner = EasyScanner.new(Pattern, Types, Keywords)
+
+      tokens = []
+      scanner.scan(input) {|t|
+	tokens << t
+      }
+
+      new_tokens = []
+      tokens.each {|t|
+	parse_token(t, new_tokens)
+	new_tokens << t
+      }
+
+      new_tokens.each {|t|
+	yield t
+      }
+    end
+
+    LangScan.register(self)
+  end
+end

data/lib/langscan/d.rb

@@ -0,0 +1,201 @@
+#
+# d.rb - a D module of LangScan
+#
+# Copyright (C) 2005 Shinichiro Hamaji <hamaji@nii.ac.jp>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'langscan/d/d'
+require 'langscan/_common'
+require 'langscan/_pairmatcher'
+
+module LangScan
+  module D
+    module_function
+
+    DEFAULT_ALLOW_TEMPLATE_PARAMETERS = 3
+
+    def name
+      "D"
+    end
+
+    def abbrev
+      "d"
+    end
+
+    def extnames
+      [".d"]
+    end
+
+    def is_fun?(fun)
+      fun.type == :ident && IdentType[fun.text] == :ident
+    end
+
+    def is_fundef?(pair)
+      1 <= pair.after_close_length &&
+        (pair.around_close(1).type == :punct &&
+           /\A\{\z/ =~ pair.around_close(1).text)
+    end
+
+    def is_template?(pair)
+      t = pair.around_open(-2)
+      if (t && (t.text == 'template'))
+        return true
+      end
+
+      return false
+    end
+
+    def is_new?(pair)
+      t = pair.around_open(-2)
+      if (t && (t.text == 'new'))
+        return true
+      end
+
+      return false
+    end
+
+    def is_fplike?(pair)
+      if pair.around_open(1) && pair.around_open(1).text == '*'
+        pair.around_close(1) && pair.around_close(1).text == '('
+        # type (*arg)());
+        return true
+      end
+
+      t = pair.around_open(-2)
+      if (t && (t.text == 'delegate' || t.text == 'function'))
+        return true
+      end
+
+      return false
+    end
+
+    def get_funtype(o)
+      return :fundecl if (!o)
+
+      return :funcall if (o.pair_def[1] != '{')
+
+      l = o.around_open(-1)
+
+      return :fundecl if (!l || l.type == :classdef)
+
+      lt = l.text
+
+      if (lt == 'in' || lt == 'out' || lt == 'body' || lt == 'unittest')
+        return :funcall
+      end
+
+      if (lt == ')')
+        i = -2
+        while (o.around_open(i) && o.around_open(i).text != '(')
+          i -= 1
+        end
+
+        # mismatched paren not reached, this is if or for or...
+        return :funcall if (!o.around_open(i))
+
+        l2 = o.around_open(i-1)
+        # this is if or for or...
+        return :funcall if (!l2)
+
+        l2t = l2.text
+
+        if (l2t != 'version' && l2t != 'extern' && l2t != 'debug' && l2t != 'pragma')
+          l3 = o.around_open(i-2)
+          return :fundecl if (l3 && l3.text == 'template')
+
+          return :funcall
+        end
+
+      end
+
+      get_funtype(o.outer)
+    end
+
+    # LangScan::D.scan iterates over D program.
+    # It yields for each element which is interested by gonzui. 
+    #
+    def scan(input, atp = DEFAULT_ALLOW_TEMPLATE_PARAMETERS, &block)
+      sorter = PairMatcher.fragmentsorter(block)
+      scan_unsorted(input, atp, &sorter)
+    end
+
+    def scan_unsorted(input, atp = DEFAULT_ALLOW_TEMPLATE_PARAMETERS, &block)
+      before_num = 3 + atp * 2
+      before_num = 4 if (before_num < 4)
+
+      pm = LangScan::PairMatcher.new(before_num,2,2,2)
+      pm.define_intertoken_fragment :space, nil
+      pm.define_intertoken_fragment :preproc, nil
+      pm.define_intertoken_fragment :comment, nil
+      pm.define_pair :paren, :punct, "(", :punct, ")"
+      pm.define_pair :brace, :punct, "{", :punct, "}"
+      pm.define_pair :bracket, :punct, "[", :punct, "]"
+      pm.parse(LangScan::D::Tokenizer.new(input), lambda {|f|
+        if f.type == :ident
+          f.type = IdentType[f.text]
+        end
+        yield f
+      }) {|pair|
+        if pair.pair_type == :paren
+          if 1 <= pair.before_open_length
+            fun = pair.around_open(-1)
+            if is_fun?(fun)
+              # ident(...)
+              if is_new?(pair)
+                fun.type = :classref
+              elsif is_template?(pair)
+                fun.type = :moduledef
+              elsif is_fplike?(pair)
+              elsif is_fundef?(pair)
+                # name(...) { ... }
+                fun.type = :fundef
+              else
+                outer = pair.outer
+                fun.type = get_funtype(outer)
+              end
+            end
+          end
+        end
+      }
+    end
+
+    Keywords = %w(
+      abstract alias align asm assert auto bit body break byte
+      case cast catch cdouble cent cfloat char class const continue creal
+      dchar debug default delegate delete deprecated do double
+      else enum export extern false final finally float for foreach function
+      goto idouble if ifloat import in inout int interface invariant ireal is
+      long mixin module new null out override package pragma private protected
+      public real return short static struct super switch synchronized
+      template this ~this throw true try typedef typeid typeof ubyte
+      ucent uint ulong union unittest ushort version void volatile
+      wchar while with
+    )
+    KeywordsHash = {}
+    Keywords.each {|k| KeywordsHash[k] = k }
+
+    Types = %w(bool char double float int long short void)
+    TypesHash = {}
+    Types.each {|k| TypesHash[k] = k }
+
+    IdentType = Hash.new(:ident)
+    Keywords.each {|k| IdentType[k] = :keyword }
+    Types.each {|k| IdentType[k] = :type }
+
+    # for debug
+    def D.each_fragment(input)
+      tokenizer = LangScan::D::Tokenizer.new(input)
+      while t = tokenizer.get_token
+        yield t
+      end
+    end
+
+    LangScan.register(self)
+  end
+end
+