langscan 1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (168) hide show
  1. data/AUTHORS.txt +19 -0
  2. data/History.txt +126 -0
  3. data/Manifest.txt +167 -0
  4. data/README.rdoc +89 -0
  5. data/Rakefile +40 -0
  6. data/ext/langscan/_make_c.rb +20 -0
  7. data/ext/langscan/_make_h.rb +30 -0
  8. data/ext/langscan/_template.c +134 -0
  9. data/ext/langscan/_template.h +53 -0
  10. data/ext/langscan/c/c/Makefile +157 -0
  11. data/ext/langscan/c/c/c.c +134 -0
  12. data/ext/langscan/c/c/c.h +66 -0
  13. data/ext/langscan/c/c/ctok.c +4622 -0
  14. data/ext/langscan/c/c/ctok.l +212 -0
  15. data/ext/langscan/c/c/extconf.rb +3 -0
  16. data/ext/langscan/c/c/modulename.txt +1 -0
  17. data/ext/langscan/c/c/tokenlist.txt +13 -0
  18. data/ext/langscan/csharp/csharp/Makefile +157 -0
  19. data/ext/langscan/csharp/csharp/csharp.c +134 -0
  20. data/ext/langscan/csharp/csharp/csharp.h +65 -0
  21. data/ext/langscan/csharp/csharp/csharptok.c +2965 -0
  22. data/ext/langscan/csharp/csharp/csharptok.l +200 -0
  23. data/ext/langscan/csharp/csharp/extconf.rb +3 -0
  24. data/ext/langscan/csharp/csharp/modulename.txt +1 -0
  25. data/ext/langscan/csharp/csharp/tokenlist.txt +12 -0
  26. data/ext/langscan/d/d/Makefile +157 -0
  27. data/ext/langscan/d/d/d.c +134 -0
  28. data/ext/langscan/d/d/d.h +64 -0
  29. data/ext/langscan/d/d/dtok.c +5461 -0
  30. data/ext/langscan/d/d/dtok.l +282 -0
  31. data/ext/langscan/d/d/extconf.rb +3 -0
  32. data/ext/langscan/d/d/modulename.txt +1 -0
  33. data/ext/langscan/d/d/tokenlist.txt +11 -0
  34. data/ext/langscan/elisp/elisp/Makefile +157 -0
  35. data/ext/langscan/elisp/elisp/elisp.c +134 -0
  36. data/ext/langscan/elisp/elisp/elisp.h +62 -0
  37. data/ext/langscan/elisp/elisp/elisptok.c +2101 -0
  38. data/ext/langscan/elisp/elisp/elisptok.l +151 -0
  39. data/ext/langscan/elisp/elisp/extconf.rb +3 -0
  40. data/ext/langscan/elisp/elisp/modulename.txt +1 -0
  41. data/ext/langscan/elisp/elisp/tokenlist.txt +9 -0
  42. data/ext/langscan/java/java/Makefile +157 -0
  43. data/ext/langscan/java/java/extconf.rb +3 -0
  44. data/ext/langscan/java/java/java.c +134 -0
  45. data/ext/langscan/java/java/java.h +64 -0
  46. data/ext/langscan/java/java/javatok.c +2090 -0
  47. data/ext/langscan/java/java/javatok.l +155 -0
  48. data/ext/langscan/java/java/modulename.txt +1 -0
  49. data/ext/langscan/java/java/tokenlist.txt +11 -0
  50. data/ext/langscan/javascript/javascript/Makefile +157 -0
  51. data/ext/langscan/javascript/javascript/extconf.rb +3 -0
  52. data/ext/langscan/javascript/javascript/javascript.c +134 -0
  53. data/ext/langscan/javascript/javascript/javascript.h +63 -0
  54. data/ext/langscan/javascript/javascript/javascripttok.c +2051 -0
  55. data/ext/langscan/javascript/javascript/javascripttok.l +147 -0
  56. data/ext/langscan/javascript/javascript/modulename.txt +1 -0
  57. data/ext/langscan/javascript/javascript/tokenlist.txt +10 -0
  58. data/ext/langscan/pairmatcher/pairmatcher/Makefile +157 -0
  59. data/ext/langscan/pairmatcher/pairmatcher/extconf.rb +3 -0
  60. data/ext/langscan/pairmatcher/pairmatcher/pairmatcher.c +890 -0
  61. data/ext/langscan/php/php/Makefile +157 -0
  62. data/ext/langscan/php/php/extconf.rb +3 -0
  63. data/ext/langscan/php/php/modulename.txt +1 -0
  64. data/ext/langscan/php/php/php.c +134 -0
  65. data/ext/langscan/php/php/php.h +64 -0
  66. data/ext/langscan/php/php/phptok.c +2406 -0
  67. data/ext/langscan/php/php/phptok.l +212 -0
  68. data/ext/langscan/php/php/tokenlist.txt +11 -0
  69. data/ext/langscan/post-distclean.rb +21 -0
  70. data/ext/langscan/pre-config.rb +57 -0
  71. data/ext/langscan/python/python/Makefile +157 -0
  72. data/ext/langscan/python/python/extconf.rb +3 -0
  73. data/ext/langscan/python/python/modulename.txt +1 -0
  74. data/ext/langscan/python/python/python.c +134 -0
  75. data/ext/langscan/python/python/python.h +61 -0
  76. data/ext/langscan/python/python/pythontok.c +2102 -0
  77. data/ext/langscan/python/python/pythontok.l +155 -0
  78. data/ext/langscan/python/python/tokenlist.txt +8 -0
  79. data/ext/langscan/ruby/compat/ripper/Makefile +158 -0
  80. data/ext/langscan/ruby/compat/ripper/depend +1 -0
  81. data/ext/langscan/ruby/compat/ripper/extconf.rb +4 -0
  82. data/ext/langscan/ruby/compat/ripper/include/eventids1.c +251 -0
  83. data/ext/langscan/ruby/compat/ripper/include/eventids2.c +277 -0
  84. data/ext/langscan/ruby/compat/ripper/include/lex.c +138 -0
  85. data/ext/langscan/ruby/compat/ripper/ripper.c +14420 -0
  86. data/ext/langscan/scheme/scheme/Makefile +157 -0
  87. data/ext/langscan/scheme/scheme/extconf.rb +3 -0
  88. data/ext/langscan/scheme/scheme/modulename.txt +1 -0
  89. data/ext/langscan/scheme/scheme/scheme.c +134 -0
  90. data/ext/langscan/scheme/scheme/scheme.h +60 -0
  91. data/ext/langscan/scheme/scheme/schemetok.c +2447 -0
  92. data/ext/langscan/scheme/scheme/schemetok.l +177 -0
  93. data/ext/langscan/scheme/scheme/tokenlist.txt +7 -0
  94. data/ext/langscan/sh/sh/Makefile +157 -0
  95. data/ext/langscan/sh/sh/extconf.rb +3 -0
  96. data/ext/langscan/sh/sh/modulename.txt +1 -0
  97. data/ext/langscan/sh/sh/sh.c +134 -0
  98. data/ext/langscan/sh/sh/sh.h +61 -0
  99. data/ext/langscan/sh/sh/shtok.c +2470 -0
  100. data/ext/langscan/sh/sh/shtok.l +325 -0
  101. data/ext/langscan/sh/sh/tokenlist.txt +8 -0
  102. data/lib/langscan.rb +124 -0
  103. data/lib/langscan/_common.rb +50 -0
  104. data/lib/langscan/_easyscanner.rb +78 -0
  105. data/lib/langscan/_pairmatcher.rb +46 -0
  106. data/lib/langscan/_type.rb +125 -0
  107. data/lib/langscan/autoconf.rb +51 -0
  108. data/lib/langscan/automake.rb +51 -0
  109. data/lib/langscan/brainfuck.rb +48 -0
  110. data/lib/langscan/c.rb +144 -0
  111. data/lib/langscan/csharp.rb +101 -0
  112. data/lib/langscan/css.rb +109 -0
  113. data/lib/langscan/d.rb +201 -0
  114. data/lib/langscan/eiffel.rb +167 -0
  115. data/lib/langscan/elisp.rb +132 -0
  116. data/lib/langscan/io.rb +84 -0
  117. data/lib/langscan/java.rb +95 -0
  118. data/lib/langscan/javascript.rb +97 -0
  119. data/lib/langscan/lua.rb +116 -0
  120. data/lib/langscan/ocaml.rb +298 -0
  121. data/lib/langscan/ocaml/camlexer.ml +28 -0
  122. data/lib/langscan/ocaml/lexer.mll +230 -0
  123. data/lib/langscan/ocaml/types.ml +36 -0
  124. data/lib/langscan/perl.rb +87 -0
  125. data/lib/langscan/perl/tokenizer.pl +231 -0
  126. data/lib/langscan/php.rb +80 -0
  127. data/lib/langscan/python.rb +101 -0
  128. data/lib/langscan/rpmspec.rb +71 -0
  129. data/lib/langscan/ruby.rb +164 -0
  130. data/lib/langscan/ruby/compat/README +5 -0
  131. data/lib/langscan/ruby/compat/ripper.rb +4 -0
  132. data/lib/langscan/ruby/compat/ripper/core.rb +918 -0
  133. data/lib/langscan/ruby/compat/ripper/filter.rb +70 -0
  134. data/lib/langscan/ruby/compat/ripper/lexer.rb +179 -0
  135. data/lib/langscan/ruby/compat/ripper/sexp.rb +100 -0
  136. data/lib/langscan/scheme.rb +160 -0
  137. data/lib/langscan/sh.rb +116 -0
  138. data/lib/langscan/text.rb +37 -0
  139. data/metaconfig +2 -0
  140. data/script/console +10 -0
  141. data/script/destroy +14 -0
  142. data/script/generate +14 -0
  143. data/script/makemanifest.rb +21 -0
  144. data/setup.rb +1604 -0
  145. data/tasks/extconf.rake +13 -0
  146. data/tasks/extconf/langscan.rake +42 -0
  147. data/test/langscan/brainfuck/test/test_scan.rb +55 -0
  148. data/test/langscan/c/test/test_scan.rb +216 -0
  149. data/test/langscan/c/test/test_token.rb +41 -0
  150. data/test/langscan/csharp/test/test_scan.rb +157 -0
  151. data/test/langscan/css/test/test_css.rb +79 -0
  152. data/test/langscan/d/test/test_scan.rb +233 -0
  153. data/test/langscan/d/test/test_token.rb +205 -0
  154. data/test/langscan/eiffel/test/test_eiffel.rb +95 -0
  155. data/test/langscan/elisp/test/test_elisp.rb +177 -0
  156. data/test/langscan/io/test/test_io.rb +79 -0
  157. data/test/langscan/java/test/test_java.rb +74 -0
  158. data/test/langscan/javascript/test/test_javascript.rb +39 -0
  159. data/test/langscan/lua/test/test_lua.rb +69 -0
  160. data/test/langscan/ocaml/test/test_ocaml.rb +161 -0
  161. data/test/langscan/php/test/test_scan.rb +138 -0
  162. data/test/langscan/python/test/test_scan.rb +105 -0
  163. data/test/langscan/rpmspec/test/test_rpmspec.rb +51 -0
  164. data/test/langscan/ruby/test/test_scan.rb +71 -0
  165. data/test/langscan/scheme/test/test_scan.rb +198 -0
  166. data/test/test_helper.rb +7 -0
  167. data/test/test_langscan.rb +123 -0
  168. metadata +296 -0
@@ -0,0 +1,167 @@
1
+ #
2
+ # eiffel.rb - a Eiffel module of LangScan
3
+ #
4
+ # Copyright (C) 2005 Shinichiro Hamaji <hamaji@nii.ac.jp>
5
+ # All rights reserved.
6
+ # This is free software with ABSOLUTELY NO WARRANTY.
7
+ #
8
+ # You can redistribute it and/or modify it under the terms of
9
+ # the GNU General Public License version 2.
10
+ #
11
+
12
+ require 'langscan/_easyscanner'
13
+ require 'langscan/_pairmatcher'
14
+
15
+ module LangScan
16
+ module Eiffel
17
+ module_function
18
+ def name
19
+ "Eiffel"
20
+ end
21
+
22
+ def abbrev
23
+ "eiffel"
24
+ end
25
+
26
+ def extnames
27
+ [".e"]
28
+ end
29
+
30
+ Pattern = [
31
+ [:comment, '--.*'],
32
+ [:character, "'%.'"],
33
+ [:character, "'[^%]'"],
34
+ [:string, '""'],
35
+ [:string, '"', '[^%]"'],
36
+ [:floating, '(?:\\d[\\d_]*)?\\.(?:\\d[\\d_]*)?(?:[eE][+-]?(?:\\d[\\d_]*))?'],
37
+ [:integer, '\\d[\\d_]*'],
38
+ [:ident, "[a-zA-Z]\\w*"],
39
+ [:punct, '[()]'],
40
+ [:punct, '[!$*+-/^~=<>{}\\[\\];:,\\.@\\\\]+'], # rough.
41
+ ]
42
+
43
+ Types = %w(INTEGER REAL DOUBLE CHARACTER BOOLEAN)
44
+
45
+ Keywords = %w(alias all and as check class creation debug deferred do
46
+ else elseif end ensure expanded export external feature from
47
+ frozen if implies indexing infix inherit inspect invariant is
48
+ like local loop not obsolete old once or prefix redefine
49
+ rename require rescue retry select separate then undefine
50
+ until variant when xor
51
+ BIT Current False Precursor Result Strip True Unique)
52
+
53
+ class Eof < Exception
54
+ end
55
+
56
+ def check_token(tkns, index, type, name = nil)
57
+ t = tkns[index]
58
+ raise Eof.new if !t
59
+ return t.type == type && (!name || t.text == name)
60
+ end
61
+
62
+ def go_next(tkns, index, step)
63
+ for i in 0...step
64
+ index += 1
65
+ index += 1 while (check_token(tkns, index, :comment))
66
+ end
67
+ index
68
+ end
69
+
70
+ def check_token_next(tkns, index, step, type, name = nil)
71
+ index = go_next(tkns, index, step)
72
+ check_token(tkns, index, type, name)
73
+ end
74
+
75
+ def go_prev(tkns, index, step)
76
+ for i in 0...step
77
+ index -= 1
78
+ index -= 1 while (check_token(tkns, index, :comment))
79
+ end
80
+ index
81
+ end
82
+
83
+ def check_token_prev(tkns, index, step, type, name = nil)
84
+ index = go_prev(tkns, index, step)
85
+ check_token(tkns, index, type, name)
86
+ end
87
+
88
+ def back_token(tkns, i, type, name)
89
+ i -= 1 while (i >= 0 && !check_token(tkns, i, type, name))
90
+ i
91
+ end
92
+
93
+ def scan(input, &block)
94
+ tkns = Array.new
95
+ scanner = EasyScanner.new(Pattern, Types, Keywords)
96
+ scanner.scan(input) do |f|
97
+ if f.type == :ident
98
+ f.type = IdentType[f.text.downcase]
99
+ elsif f.type == :floating && f.text == '.'
100
+ f.type = :punct
101
+ end
102
+ tkns << f
103
+ end
104
+
105
+ begin
106
+ i = 0
107
+ while (i < tkns.size)
108
+ if (check_token(tkns, i, :punct, ')'))
109
+ j = back_token(tkns, i-1, :punct, '(')
110
+ j = go_prev(tkns, j, 1)
111
+ if (j >= 0 && check_token(tkns, j, :ident))
112
+ if (check_token_next(tkns, i, 1, :keyword, 'is') ||
113
+ check_token_next(tkns, i, 1, :punct, ':'))
114
+ tkns[j].type = :fundef
115
+ i += 1 while (!check_token(tkns, i, :keyword))
116
+ else
117
+ tkns[j].type = :funcall
118
+ end
119
+ end
120
+ elsif (check_token(tkns, i, :keyword, 'is'))
121
+ if (check_token_next(tkns, i, 1, :keyword, 'do') ||
122
+ check_token_next(tkns, i, 1, :keyword, 'local') ||
123
+ check_token_next(tkns, i, 1, :keyword, 'require') ||
124
+ check_token_next(tkns, i, 1, :keyword, 'once') ||
125
+ check_token_next(tkns, i, 1, :keyword, 'external'))
126
+ if (check_token_prev(tkns, i, 3, :ident) &&
127
+ check_token_prev(tkns, i, 2, :punct, ':') &&
128
+ (check_token_prev(tkns, i, 1, :ident) ||
129
+ check_token_prev(tkns, i, 1, :type)))
130
+ tkns[go_prev(tkns, i, 3)].type = :fundef
131
+ elsif (check_token_prev(tkns, i, 1, :ident))
132
+ tkns[go_prev(tkns, i, 1)].type = :fundef
133
+ end
134
+ end
135
+ elsif (check_token(tkns, i, :keyword, 'class'))
136
+ if (check_token_next(tkns, i, 1, :ident))
137
+ tkns[go_next(tkns, i, 1)].type = :classdef
138
+ end
139
+ elsif (check_token(tkns, i, :keyword, 'rename'))
140
+ while (check_token_next(tkns, i, 1, :ident) &&
141
+ check_token_next(tkns, i, 2, :keyword, 'as') &&
142
+ check_token_next(tkns, i, 3, :ident))
143
+ tkns[i = go_next(tkns, i, 3)].type = :fundef
144
+ i = go_next(tkns, i, 1)
145
+ break if (!check_token(tkns, i, :punct, ','))
146
+ end
147
+ end
148
+
149
+ i += 1
150
+
151
+ end
152
+ rescue Eof
153
+ end
154
+
155
+ tkns.each do |f|
156
+ yield f
157
+ end
158
+
159
+ end
160
+
161
+ IdentType = Hash.new(:ident)
162
+ Keywords.each {|k| IdentType[k.downcase] = :keyword }
163
+ Types.each {|k| IdentType[k] = :type }
164
+
165
+ LangScan.register(self)
166
+ end
167
+ end
@@ -0,0 +1,132 @@
1
+ #
2
+ # elisp.rb - a elisp module of LangScan
3
+ #
4
+ # Copyright (C) 2005 Keisuke Nishida <knishida@open-cobol.org>
5
+ # 2005 Kenichi Ishibashi <bashi at dream.ie.ariake-nct.ac.jp>
6
+ # All rights reserved.
7
+ # This is free software with ABSOLUTELY NO WARRANTY.
8
+ #
9
+ # You can redistribute it and/or modify it under the terms of
10
+ # the GNU General Public License version 2.
11
+ #
12
+
13
+ require 'langscan/elisp/elisp'
14
+ require 'langscan/_common'
15
+ require 'langscan/_pairmatcher'
16
+
17
+ module LangScan
18
+ module EmacsLisp
19
+ module_function
20
+ def name
21
+ "Emacs Lisp"
22
+ end
23
+
24
+ def abbrev
25
+ "elisp"
26
+ end
27
+
28
+ def extnames
29
+ [".el"]
30
+ end
31
+
32
+ # LangScan::EmacsLisp.scan iterates over Emacs Lisp program.
33
+ # It yields for each Fragment.
34
+ def scan(input, &block)
35
+ sorter = PairMatcher.fragmentsorter(block)
36
+ scan_unsorted(input, &sorter)
37
+ end
38
+
39
+ def scan_unsorted(input, &block)
40
+ pm = LangScan::PairMatcher.new(2,2,2,2)
41
+ pm.define_intertoken_fragment :space, nil
42
+ pm.define_intertoken_fragment :comment, nil
43
+ pm.define_pair :paren, :punct, "(", :punct, ")"
44
+ reporter = lambda {|f|
45
+ if (f.type == :ident || f.type == :funcall) && KeywordsHash[f.text]
46
+ f.type = :keyword
47
+ end
48
+ if f.type == :quote_chars
49
+ f.type = :punct
50
+ end
51
+ yield f
52
+ }
53
+ pm.parse(LangScan::EmacsLisp::Tokenizer.new(input), reporter) {|list|
54
+ if (list.around_open(1).text == "defun" || list.around_open(1).text == "defmacro") && list.around_open(2).type == :ident
55
+ list.around_open(2).type = :fundef
56
+ end
57
+ if list.around_open(1).type == :ident && funcall_list?(list)
58
+ list.around_open(1).type = :funcall
59
+ end
60
+ }
61
+ end
62
+
63
+ def funcall_list?(list)
64
+ if list.before_open_length == 0
65
+ return true
66
+ end
67
+ if list.before_open_length >= 2 && list.around_open(-2).text == "defun"
68
+ return false
69
+ end
70
+ if NotFuncallWordsHash[list.around_open(-1).text]
71
+ return false
72
+ end
73
+ if quote_list?(list)
74
+ return false
75
+ end
76
+ outer = list.outer
77
+ second_outer = outer.outer unless outer == nil
78
+ if second_outer and NotFuncall2ndOuterWordsHash[second_outer.around_open(1).text]
79
+ if NotFuncall2ndOuterWordsHash[outer.around_open(-1).text]
80
+ return false
81
+ end
82
+ end
83
+ return true
84
+ end
85
+
86
+ def quote_nestlevel(str)
87
+ str.count("`") - str.count(",")
88
+ end
89
+
90
+ def quote_list?(list)
91
+ l = list
92
+ nest = 0
93
+ while l
94
+ if l.before_open_length >= 1
95
+ before = l.around_open(-1)
96
+ if before.type == :quote_chars
97
+ return true if before.text.include?("'")
98
+ nest = nest + quote_nestlevel(before.text)
99
+ end
100
+ end
101
+ if l.after_open_length >= 1
102
+ after = l.around_open(1)
103
+ if after.text == 'quote'
104
+ return true
105
+ elsif after.text == 'backquote'
106
+ nest = nest + 1
107
+ elsif after.text == 'unquote'
108
+ nest = nest - 1
109
+ end
110
+ end
111
+ l = l.outer
112
+ end
113
+ return nest > 0
114
+ end
115
+
116
+ Keywords = %w(
117
+ defun defvar defmacro defgroup defcustom defconst and or
118
+ condition-case function interactive require quote backquote
119
+ lambda prog1 prog2 progn let let* if when unless cond catch throw
120
+ save-current-buffer save-excursion save-restriction save-window-excurtion
121
+ setq setq-default track-mouse unwind-protect while
122
+ with-output-to-temp-buffer t nil
123
+ )
124
+ KeywordsHash = {}
125
+ Keywords.each {|k| KeywordsHash[k] = k }
126
+
127
+ NotFuncallWordsHash = {"lambda" => "lambda"}
128
+ NotFuncall2ndOuterWordsHash = {"let" => "let", "let*" => "let*"}
129
+
130
+ LangScan.register(self)
131
+ end
132
+ end
@@ -0,0 +1,84 @@
1
+ #
2
+ # io.rb - a Io module of LangScan
3
+ #
4
+ # Copyright (C) 2005 Shinichiro Hamaji <hamaji@nii.ac.jp>
5
+ # All rights reserved.
6
+ # This is free software with ABSOLUTELY NO WARRANTY.
7
+ #
8
+ # You can redistribute it and/or modify it under the terms of
9
+ # the GNU General Public License version 2.
10
+ #
11
+
12
+ require 'langscan/_easyscanner'
13
+
14
+ module LangScan
15
+ module Io
16
+ module_function
17
+ def name
18
+ "Io"
19
+ end
20
+
21
+ def abbrev
22
+ "io"
23
+ end
24
+
25
+ def extnames
26
+ [".io"]
27
+ end
28
+
29
+ Pattern = [
30
+ [:comment, '/\\*', '\\*/'],
31
+ [:comment, '//.*'],
32
+ [:comment, '#.*'],
33
+ [:string, '"""', '[^\\\\]"""'],
34
+ [:string, '""'],
35
+ [:string, '"', '[^\\\\]"'],
36
+ [:floating, '\\d*(?:(?:\\.\\d+)?(?:e-?\\d+)|(?:\\.\\d+))'],
37
+ [:integer, '0[xX][\\dA-Fa-f]+'],
38
+ [:integer, '\\d+'],
39
+ [:ident, "\\w*[a-zA-Z_]\\w*"],
40
+ [:punct, '[:\\.\'~!@$%=^&*-+|<>?/]+'],
41
+ [:punct, '[\\[\\]{}();,]'],
42
+ ]
43
+
44
+ Types = []
45
+
46
+ Keywords = %w()
47
+
48
+ def parse_token(t, new_tokens)
49
+ if t.type == :ident
50
+ t.type = :funcall
51
+ end
52
+
53
+ last_token = new_tokens.last
54
+ return if last_token.nil?
55
+
56
+ return unless t.type == :punct and last_token.type == :funcall
57
+
58
+ if t.text == ':=' || t.text == '='
59
+ last_token.type = :fundef
60
+ end
61
+ end
62
+
63
+ def scan(input, &block)
64
+ scanner = EasyScanner.new(Pattern, Types, Keywords)
65
+
66
+ tokens = []
67
+ scanner.scan(input) {|t|
68
+ tokens << t
69
+ }
70
+
71
+ new_tokens = []
72
+ tokens.each {|t|
73
+ parse_token(t, new_tokens)
74
+ new_tokens << t
75
+ }
76
+
77
+ new_tokens.each {|t|
78
+ yield t
79
+ }
80
+ end
81
+
82
+ LangScan.register(self)
83
+ end
84
+ end
@@ -0,0 +1,95 @@
1
+ #
2
+ # java.rb - a Java module of LangScan
3
+ #
4
+ # Copyright (C) 2004-2005 Keisuke Nishida <knishida@open-cobol.org>
5
+ # All rights reserved.
6
+ # This is free software with ABSOLUTELY NO WARRANTY.
7
+ #
8
+ # You can redistribute it and/or modify it under the terms of
9
+ # the GNU General Public License version 2.
10
+ #
11
+
12
+ require 'langscan/java/java'
13
+ require 'langscan/_common'
14
+ require 'langscan/_pairmatcher'
15
+
16
+ module LangScan
17
+ module Java
18
+ module_function
19
+ def name
20
+ "Java"
21
+ end
22
+
23
+ def abbrev
24
+ "java"
25
+ end
26
+
27
+ def extnames
28
+ [".java"]
29
+ end
30
+
31
+ # LangScan::Java.scan iterates over Java program.
32
+ # It yields for each Fragment.
33
+ def scan(input, &block)
34
+ sorter = PairMatcher.fragmentsorter(block)
35
+ scan_unsorted(input, &sorter)
36
+ end
37
+
38
+ def scan_unsorted(input, &block)
39
+ pm = LangScan::PairMatcher.new(1,0,0,1)
40
+ pm.define_intertoken_fragment :space, nil
41
+ pm.define_intertoken_fragment :comment, nil
42
+ pm.define_pair :paren, :punct, "(", :punct, ")"
43
+ pm.define_pair :brace, :punct, "{", :punct, "}"
44
+ pm.define_pair :bracket, :punct, "[", :punct, "]"
45
+ pm.parse(LangScan::Java::Tokenizer.new(input), lambda {|f|
46
+ if f.type == :ident
47
+ f.type = IdentType[f.text]
48
+ end
49
+ yield f
50
+ }) {|pair|
51
+ if pair.pair_type == :paren &&
52
+ 1 <= pair.before_open_length &&
53
+ pair.around_open(-1).type == :ident && IdentType[pair.around_open(-1).text] == :ident
54
+ before_open_token = pair.around_open(-1)
55
+ if !KeywordsHash[before_open_token.text]
56
+ if !(outer = pair.outer) || !outer.outer
57
+ if 1 <= pair.after_close_length &&
58
+ (pair.around_close(1).type == :punct &&
59
+ pair.around_close(1).text == '{' ||
60
+ pair.around_close(1).type == :ident &&
61
+ pair.around_close(1).text == 'throws')
62
+ before_open_token.type = :fundef
63
+ else
64
+ before_open_token.type = :funcall
65
+ end
66
+ else
67
+ before_open_token.type = :funcall
68
+ end
69
+ end
70
+ end
71
+ }
72
+ end
73
+
74
+ Keywords = %w(
75
+ abstract boolean break byte case catch char class const continue default
76
+ do double else extends final finally float for goto if implements import
77
+ int interface long native new package private protected public return
78
+ short static strictfp super switch synchronized this throw throws
79
+ transient try void volatile while true false null
80
+ )
81
+ KeywordsHash = {}
82
+ Keywords.each {|k| KeywordsHash[k] = k }
83
+
84
+ Types = %w(boolean byte char double float int long short void)
85
+ TypesHash = {}
86
+ Types.each {|k| TypesHash[k] = k }
87
+
88
+ IdentType = Hash.new(:ident)
89
+ Keywords.each {|k| IdentType[k] = :keyword }
90
+ Types.each {|k| IdentType[k] = :type }
91
+
92
+ LangScan.register(self)
93
+ end
94
+ end
95
+