langscan 1.2

data/lib/langscan/eiffel.rb

@@ -0,0 +1,167 @@
+#
+# eiffel.rb - a Eiffel module of LangScan
+#
+# Copyright (C) 2005 Shinichiro Hamaji <hamaji@nii.ac.jp>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'langscan/_easyscanner'
+require 'langscan/_pairmatcher'
+
+module LangScan
+  module Eiffel
+    module_function
+    def name
+      "Eiffel"
+    end
+
+    def abbrev
+      "eiffel"
+    end
+
+    def extnames
+      [".e"]
+    end
+
+    Pattern = [
+      [:comment, '--.*'],
+      [:character, "'%.'"],
+      [:character, "'[^%]'"],
+      [:string, '""'],
+      [:string, '"', '[^%]"'],
+      [:floating, '(?:\\d[\\d_]*)?\\.(?:\\d[\\d_]*)?(?:[eE][+-]?(?:\\d[\\d_]*))?'],
+      [:integer, '\\d[\\d_]*'],
+      [:ident, "[a-zA-Z]\\w*"],
+      [:punct, '[()]'],
+      [:punct, '[!$*+-/^~=<>{}\\[\\];:,\\.@\\\\]+'],  # rough.
+    ]
+
+    Types = %w(INTEGER REAL DOUBLE CHARACTER BOOLEAN)
+
+    Keywords = %w(alias all and as check class creation debug deferred do
+                  else elseif end ensure expanded export external feature from
+                  frozen if implies indexing infix inherit inspect invariant is
+                  like local loop not obsolete old once or prefix redefine
+                  rename require rescue retry select separate then undefine
+                  until variant when xor
+                  BIT Current False Precursor Result Strip True Unique)
+
+    class Eof < Exception
+    end
+
+    def check_token(tkns, index, type, name = nil)
+      t = tkns[index]
+      raise Eof.new if !t
+      return t.type == type && (!name || t.text == name)
+    end
+
+    def go_next(tkns, index, step)
+      for i in 0...step
+        index += 1
+        index += 1 while (check_token(tkns, index, :comment))
+      end
+      index
+    end
+
+    def check_token_next(tkns, index, step, type, name = nil)
+      index = go_next(tkns, index, step)
+      check_token(tkns, index, type, name)
+    end
+
+    def go_prev(tkns, index, step)
+      for i in 0...step
+        index -= 1
+        index -= 1 while (check_token(tkns, index, :comment))
+      end
+      index
+    end
+
+    def check_token_prev(tkns, index, step, type, name = nil)
+      index = go_prev(tkns, index, step)
+      check_token(tkns, index, type, name)
+    end
+
+    def back_token(tkns, i, type, name)
+      i -= 1 while (i >= 0 && !check_token(tkns, i, type, name))
+      i
+    end
+
+    def scan(input, &block)
+      tkns = Array.new
+      scanner = EasyScanner.new(Pattern, Types, Keywords)
+      scanner.scan(input) do |f|
+        if f.type == :ident
+          f.type = IdentType[f.text.downcase]
+        elsif f.type == :floating && f.text == '.'
+          f.type = :punct
+        end
+        tkns << f
+      end
+
+      begin
+        i = 0
+        while (i < tkns.size)
+          if (check_token(tkns, i, :punct, ')'))
+            j = back_token(tkns, i-1, :punct, '(')
+            j = go_prev(tkns, j, 1)
+            if (j >= 0 && check_token(tkns, j, :ident))
+              if (check_token_next(tkns, i, 1, :keyword, 'is') ||
+                  check_token_next(tkns, i, 1, :punct, ':'))
+                tkns[j].type = :fundef
+                i += 1 while (!check_token(tkns, i, :keyword))
+              else
+                tkns[j].type = :funcall
+              end
+            end
+          elsif (check_token(tkns, i, :keyword, 'is'))
+            if (check_token_next(tkns, i, 1, :keyword, 'do') ||
+                check_token_next(tkns, i, 1, :keyword, 'local') ||
+                check_token_next(tkns, i, 1, :keyword, 'require') ||
+                check_token_next(tkns, i, 1, :keyword, 'once') ||
+                check_token_next(tkns, i, 1, :keyword, 'external'))
+              if (check_token_prev(tkns, i, 3, :ident) &&
+                  check_token_prev(tkns, i, 2, :punct, ':') &&
+                  (check_token_prev(tkns, i, 1, :ident) ||
+                   check_token_prev(tkns, i, 1, :type)))
+                tkns[go_prev(tkns, i, 3)].type = :fundef
+              elsif (check_token_prev(tkns, i, 1, :ident))
+                tkns[go_prev(tkns, i, 1)].type = :fundef
+              end
+            end
+          elsif (check_token(tkns, i, :keyword, 'class'))
+            if (check_token_next(tkns, i, 1, :ident))
+              tkns[go_next(tkns, i, 1)].type = :classdef
+            end
+          elsif (check_token(tkns, i, :keyword, 'rename'))
+            while (check_token_next(tkns, i, 1, :ident) &&
+                   check_token_next(tkns, i, 2, :keyword, 'as') &&
+                   check_token_next(tkns, i, 3, :ident))
+              tkns[i = go_next(tkns, i, 3)].type = :fundef
+              i = go_next(tkns, i, 1)
+              break if (!check_token(tkns, i, :punct, ','))
+            end
+          end
+
+          i += 1
+
+        end
+      rescue Eof
+      end
+
+      tkns.each do |f|
+        yield f
+      end
+
+    end
+
+    IdentType = Hash.new(:ident)
+    Keywords.each {|k| IdentType[k.downcase] = :keyword }
+    Types.each {|k| IdentType[k] = :type }
+
+    LangScan.register(self)
+  end
+end

data/lib/langscan/elisp.rb

@@ -0,0 +1,132 @@
+#
+# elisp.rb - a elisp module of LangScan
+#
+# Copyright (C) 2005 Keisuke Nishida <knishida@open-cobol.org>
+#               2005 Kenichi Ishibashi <bashi at dream.ie.ariake-nct.ac.jp>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'langscan/elisp/elisp'
+require 'langscan/_common'
+require 'langscan/_pairmatcher'
+
+module LangScan
+  module EmacsLisp
+    module_function
+    def name
+      "Emacs Lisp"
+    end
+
+    def abbrev
+      "elisp"
+    end
+
+    def extnames
+      [".el"]
+    end
+
+    # LangScan::EmacsLisp.scan iterates over Emacs Lisp program.
+    # It yields for each Fragment.
+    def scan(input, &block)
+      sorter = PairMatcher.fragmentsorter(block)
+      scan_unsorted(input, &sorter)
+    end
+
+    def scan_unsorted(input, &block)
+      pm = LangScan::PairMatcher.new(2,2,2,2)
+      pm.define_intertoken_fragment :space, nil
+      pm.define_intertoken_fragment :comment, nil
+      pm.define_pair :paren, :punct, "(", :punct, ")"
+      reporter = lambda {|f|
+        if (f.type == :ident || f.type == :funcall) && KeywordsHash[f.text]
+          f.type = :keyword
+        end
+        if f.type == :quote_chars
+          f.type = :punct
+        end
+        yield f
+      }
+      pm.parse(LangScan::EmacsLisp::Tokenizer.new(input), reporter) {|list|
+        if (list.around_open(1).text == "defun" || list.around_open(1).text == "defmacro") && list.around_open(2).type == :ident
+          list.around_open(2).type = :fundef
+        end
+        if list.around_open(1).type == :ident && funcall_list?(list)
+          list.around_open(1).type = :funcall
+        end
+      }
+    end
+
+    def funcall_list?(list)
+      if list.before_open_length == 0
+        return true
+      end
+      if list.before_open_length >= 2 && list.around_open(-2).text == "defun"
+        return false
+      end
+      if NotFuncallWordsHash[list.around_open(-1).text]
+        return false
+      end
+      if quote_list?(list)
+        return false
+      end
+      outer = list.outer
+      second_outer = outer.outer unless outer == nil
+      if second_outer and NotFuncall2ndOuterWordsHash[second_outer.around_open(1).text]
+        if NotFuncall2ndOuterWordsHash[outer.around_open(-1).text]
+          return false
+        end
+      end
+      return true
+    end
+
+    def quote_nestlevel(str)
+      str.count("`") - str.count(",")
+    end
+
+    def quote_list?(list)
+      l = list
+      nest = 0
+      while l
+        if l.before_open_length >= 1
+          before = l.around_open(-1)
+          if before.type == :quote_chars
+            return true if before.text.include?("'")
+            nest = nest + quote_nestlevel(before.text)
+          end
+        end
+        if l.after_open_length >= 1
+          after = l.around_open(1)
+          if after.text == 'quote'
+            return true
+          elsif after.text == 'backquote'
+            nest = nest + 1
+          elsif after.text == 'unquote'
+            nest = nest - 1
+          end
+        end
+        l = l.outer
+      end
+      return nest > 0
+    end
+
+    Keywords = %w(
+      defun defvar defmacro defgroup defcustom defconst and or
+      condition-case function interactive require quote backquote
+      lambda prog1 prog2 progn let let* if when unless cond catch throw
+      save-current-buffer save-excursion save-restriction save-window-excurtion
+      setq setq-default track-mouse unwind-protect while
+      with-output-to-temp-buffer t nil
+    )
+    KeywordsHash = {}
+    Keywords.each {|k| KeywordsHash[k] = k }
+
+    NotFuncallWordsHash = {"lambda" => "lambda"}
+    NotFuncall2ndOuterWordsHash = {"let" => "let", "let*" => "let*"}
+
+    LangScan.register(self)
+  end
+end

data/lib/langscan/io.rb

@@ -0,0 +1,84 @@
+#
+# io.rb - a Io module of LangScan
+#
+# Copyright (C) 2005 Shinichiro Hamaji <hamaji@nii.ac.jp>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'langscan/_easyscanner'
+
+module LangScan
+  module Io
+    module_function
+    def name
+      "Io"
+    end
+
+    def abbrev
+      "io"
+    end
+
+    def extnames
+      [".io"]
+    end
+
+    Pattern = [
+      [:comment, '/\\*', '\\*/'],
+      [:comment, '//.*'],
+      [:comment, '#.*'],
+      [:string, '"""', '[^\\\\]"""'],
+      [:string, '""'],
+      [:string, '"', '[^\\\\]"'],
+      [:floating, '\\d*(?:(?:\\.\\d+)?(?:e-?\\d+)|(?:\\.\\d+))'],
+      [:integer, '0[xX][\\dA-Fa-f]+'],
+      [:integer, '\\d+'],
+      [:ident, "\\w*[a-zA-Z_]\\w*"],
+      [:punct, '[:\\.\'~!@$%=^&*-+|<>?/]+'],
+      [:punct, '[\\[\\]{}();,]'],
+    ]
+
+    Types = []
+
+    Keywords = %w()
+
+    def parse_token(t, new_tokens)
+      if t.type == :ident
+        t.type = :funcall
+      end
+
+      last_token = new_tokens.last
+      return if last_token.nil?
+
+      return unless t.type == :punct and last_token.type == :funcall
+
+      if t.text == ':=' || t.text == '='
+        last_token.type = :fundef
+      end
+    end
+
+    def scan(input, &block)
+      scanner = EasyScanner.new(Pattern, Types, Keywords)
+
+      tokens = []
+      scanner.scan(input) {|t|
+        tokens << t
+      }
+
+      new_tokens = []
+      tokens.each {|t|
+        parse_token(t, new_tokens)
+        new_tokens << t
+      }
+
+      new_tokens.each {|t|
+        yield t
+      }
+    end
+
+    LangScan.register(self)
+  end
+end

data/lib/langscan/java.rb

@@ -0,0 +1,95 @@
+#
+# java.rb - a Java module of LangScan
+#
+# Copyright (C) 2004-2005 Keisuke Nishida <knishida@open-cobol.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'langscan/java/java'
+require 'langscan/_common'
+require 'langscan/_pairmatcher'
+
+module LangScan
+  module Java
+    module_function
+    def name
+      "Java"
+    end
+
+    def abbrev
+      "java"
+    end
+
+    def extnames
+      [".java"]
+    end
+
+    # LangScan::Java.scan iterates over Java program.
+    # It yields for each Fragment.
+    def scan(input, &block)
+      sorter = PairMatcher.fragmentsorter(block)
+      scan_unsorted(input, &sorter)
+    end
+
+    def scan_unsorted(input, &block)
+      pm = LangScan::PairMatcher.new(1,0,0,1)
+      pm.define_intertoken_fragment :space, nil
+      pm.define_intertoken_fragment :comment, nil
+      pm.define_pair :paren, :punct, "(", :punct, ")"
+      pm.define_pair :brace, :punct, "{", :punct, "}"
+      pm.define_pair :bracket, :punct, "[", :punct, "]"
+      pm.parse(LangScan::Java::Tokenizer.new(input), lambda {|f|
+        if f.type == :ident
+          f.type = IdentType[f.text]
+        end
+        yield f
+      }) {|pair|
+        if pair.pair_type == :paren &&
+           1 <= pair.before_open_length &&
+           pair.around_open(-1).type == :ident && IdentType[pair.around_open(-1).text] == :ident
+          before_open_token = pair.around_open(-1)
+          if !KeywordsHash[before_open_token.text]
+            if !(outer = pair.outer) || !outer.outer
+              if 1 <= pair.after_close_length &&
+                 (pair.around_close(1).type == :punct &&
+                  pair.around_close(1).text == '{' ||
+                  pair.around_close(1).type == :ident &&
+                  pair.around_close(1).text == 'throws')
+                before_open_token.type = :fundef
+              else
+                before_open_token.type = :funcall
+              end
+            else
+              before_open_token.type = :funcall
+            end
+          end
+        end
+      }
+    end
+
+    Keywords = %w(
+      abstract boolean break byte case catch char class const continue default
+      do double else extends final finally float for goto if implements import
+      int interface long native new package private protected public return
+      short static strictfp super switch synchronized this throw throws
+      transient try void volatile while true false null
+    )
+    KeywordsHash = {}
+    Keywords.each {|k| KeywordsHash[k] = k }
+
+    Types = %w(boolean byte char double float int long short void)
+    TypesHash = {}
+    Types.each {|k| TypesHash[k] = k }
+
+    IdentType = Hash.new(:ident)
+    Keywords.each {|k| IdentType[k] = :keyword }
+    Types.each {|k| IdentType[k] = :type }
+
+    LangScan.register(self)
+  end
+end
+