langscan 1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (168) hide show
  1. data/AUTHORS.txt +19 -0
  2. data/History.txt +126 -0
  3. data/Manifest.txt +167 -0
  4. data/README.rdoc +89 -0
  5. data/Rakefile +40 -0
  6. data/ext/langscan/_make_c.rb +20 -0
  7. data/ext/langscan/_make_h.rb +30 -0
  8. data/ext/langscan/_template.c +134 -0
  9. data/ext/langscan/_template.h +53 -0
  10. data/ext/langscan/c/c/Makefile +157 -0
  11. data/ext/langscan/c/c/c.c +134 -0
  12. data/ext/langscan/c/c/c.h +66 -0
  13. data/ext/langscan/c/c/ctok.c +4622 -0
  14. data/ext/langscan/c/c/ctok.l +212 -0
  15. data/ext/langscan/c/c/extconf.rb +3 -0
  16. data/ext/langscan/c/c/modulename.txt +1 -0
  17. data/ext/langscan/c/c/tokenlist.txt +13 -0
  18. data/ext/langscan/csharp/csharp/Makefile +157 -0
  19. data/ext/langscan/csharp/csharp/csharp.c +134 -0
  20. data/ext/langscan/csharp/csharp/csharp.h +65 -0
  21. data/ext/langscan/csharp/csharp/csharptok.c +2965 -0
  22. data/ext/langscan/csharp/csharp/csharptok.l +200 -0
  23. data/ext/langscan/csharp/csharp/extconf.rb +3 -0
  24. data/ext/langscan/csharp/csharp/modulename.txt +1 -0
  25. data/ext/langscan/csharp/csharp/tokenlist.txt +12 -0
  26. data/ext/langscan/d/d/Makefile +157 -0
  27. data/ext/langscan/d/d/d.c +134 -0
  28. data/ext/langscan/d/d/d.h +64 -0
  29. data/ext/langscan/d/d/dtok.c +5461 -0
  30. data/ext/langscan/d/d/dtok.l +282 -0
  31. data/ext/langscan/d/d/extconf.rb +3 -0
  32. data/ext/langscan/d/d/modulename.txt +1 -0
  33. data/ext/langscan/d/d/tokenlist.txt +11 -0
  34. data/ext/langscan/elisp/elisp/Makefile +157 -0
  35. data/ext/langscan/elisp/elisp/elisp.c +134 -0
  36. data/ext/langscan/elisp/elisp/elisp.h +62 -0
  37. data/ext/langscan/elisp/elisp/elisptok.c +2101 -0
  38. data/ext/langscan/elisp/elisp/elisptok.l +151 -0
  39. data/ext/langscan/elisp/elisp/extconf.rb +3 -0
  40. data/ext/langscan/elisp/elisp/modulename.txt +1 -0
  41. data/ext/langscan/elisp/elisp/tokenlist.txt +9 -0
  42. data/ext/langscan/java/java/Makefile +157 -0
  43. data/ext/langscan/java/java/extconf.rb +3 -0
  44. data/ext/langscan/java/java/java.c +134 -0
  45. data/ext/langscan/java/java/java.h +64 -0
  46. data/ext/langscan/java/java/javatok.c +2090 -0
  47. data/ext/langscan/java/java/javatok.l +155 -0
  48. data/ext/langscan/java/java/modulename.txt +1 -0
  49. data/ext/langscan/java/java/tokenlist.txt +11 -0
  50. data/ext/langscan/javascript/javascript/Makefile +157 -0
  51. data/ext/langscan/javascript/javascript/extconf.rb +3 -0
  52. data/ext/langscan/javascript/javascript/javascript.c +134 -0
  53. data/ext/langscan/javascript/javascript/javascript.h +63 -0
  54. data/ext/langscan/javascript/javascript/javascripttok.c +2051 -0
  55. data/ext/langscan/javascript/javascript/javascripttok.l +147 -0
  56. data/ext/langscan/javascript/javascript/modulename.txt +1 -0
  57. data/ext/langscan/javascript/javascript/tokenlist.txt +10 -0
  58. data/ext/langscan/pairmatcher/pairmatcher/Makefile +157 -0
  59. data/ext/langscan/pairmatcher/pairmatcher/extconf.rb +3 -0
  60. data/ext/langscan/pairmatcher/pairmatcher/pairmatcher.c +890 -0
  61. data/ext/langscan/php/php/Makefile +157 -0
  62. data/ext/langscan/php/php/extconf.rb +3 -0
  63. data/ext/langscan/php/php/modulename.txt +1 -0
  64. data/ext/langscan/php/php/php.c +134 -0
  65. data/ext/langscan/php/php/php.h +64 -0
  66. data/ext/langscan/php/php/phptok.c +2406 -0
  67. data/ext/langscan/php/php/phptok.l +212 -0
  68. data/ext/langscan/php/php/tokenlist.txt +11 -0
  69. data/ext/langscan/post-distclean.rb +21 -0
  70. data/ext/langscan/pre-config.rb +57 -0
  71. data/ext/langscan/python/python/Makefile +157 -0
  72. data/ext/langscan/python/python/extconf.rb +3 -0
  73. data/ext/langscan/python/python/modulename.txt +1 -0
  74. data/ext/langscan/python/python/python.c +134 -0
  75. data/ext/langscan/python/python/python.h +61 -0
  76. data/ext/langscan/python/python/pythontok.c +2102 -0
  77. data/ext/langscan/python/python/pythontok.l +155 -0
  78. data/ext/langscan/python/python/tokenlist.txt +8 -0
  79. data/ext/langscan/ruby/compat/ripper/Makefile +158 -0
  80. data/ext/langscan/ruby/compat/ripper/depend +1 -0
  81. data/ext/langscan/ruby/compat/ripper/extconf.rb +4 -0
  82. data/ext/langscan/ruby/compat/ripper/include/eventids1.c +251 -0
  83. data/ext/langscan/ruby/compat/ripper/include/eventids2.c +277 -0
  84. data/ext/langscan/ruby/compat/ripper/include/lex.c +138 -0
  85. data/ext/langscan/ruby/compat/ripper/ripper.c +14420 -0
  86. data/ext/langscan/scheme/scheme/Makefile +157 -0
  87. data/ext/langscan/scheme/scheme/extconf.rb +3 -0
  88. data/ext/langscan/scheme/scheme/modulename.txt +1 -0
  89. data/ext/langscan/scheme/scheme/scheme.c +134 -0
  90. data/ext/langscan/scheme/scheme/scheme.h +60 -0
  91. data/ext/langscan/scheme/scheme/schemetok.c +2447 -0
  92. data/ext/langscan/scheme/scheme/schemetok.l +177 -0
  93. data/ext/langscan/scheme/scheme/tokenlist.txt +7 -0
  94. data/ext/langscan/sh/sh/Makefile +157 -0
  95. data/ext/langscan/sh/sh/extconf.rb +3 -0
  96. data/ext/langscan/sh/sh/modulename.txt +1 -0
  97. data/ext/langscan/sh/sh/sh.c +134 -0
  98. data/ext/langscan/sh/sh/sh.h +61 -0
  99. data/ext/langscan/sh/sh/shtok.c +2470 -0
  100. data/ext/langscan/sh/sh/shtok.l +325 -0
  101. data/ext/langscan/sh/sh/tokenlist.txt +8 -0
  102. data/lib/langscan.rb +124 -0
  103. data/lib/langscan/_common.rb +50 -0
  104. data/lib/langscan/_easyscanner.rb +78 -0
  105. data/lib/langscan/_pairmatcher.rb +46 -0
  106. data/lib/langscan/_type.rb +125 -0
  107. data/lib/langscan/autoconf.rb +51 -0
  108. data/lib/langscan/automake.rb +51 -0
  109. data/lib/langscan/brainfuck.rb +48 -0
  110. data/lib/langscan/c.rb +144 -0
  111. data/lib/langscan/csharp.rb +101 -0
  112. data/lib/langscan/css.rb +109 -0
  113. data/lib/langscan/d.rb +201 -0
  114. data/lib/langscan/eiffel.rb +167 -0
  115. data/lib/langscan/elisp.rb +132 -0
  116. data/lib/langscan/io.rb +84 -0
  117. data/lib/langscan/java.rb +95 -0
  118. data/lib/langscan/javascript.rb +97 -0
  119. data/lib/langscan/lua.rb +116 -0
  120. data/lib/langscan/ocaml.rb +298 -0
  121. data/lib/langscan/ocaml/camlexer.ml +28 -0
  122. data/lib/langscan/ocaml/lexer.mll +230 -0
  123. data/lib/langscan/ocaml/types.ml +36 -0
  124. data/lib/langscan/perl.rb +87 -0
  125. data/lib/langscan/perl/tokenizer.pl +231 -0
  126. data/lib/langscan/php.rb +80 -0
  127. data/lib/langscan/python.rb +101 -0
  128. data/lib/langscan/rpmspec.rb +71 -0
  129. data/lib/langscan/ruby.rb +164 -0
  130. data/lib/langscan/ruby/compat/README +5 -0
  131. data/lib/langscan/ruby/compat/ripper.rb +4 -0
  132. data/lib/langscan/ruby/compat/ripper/core.rb +918 -0
  133. data/lib/langscan/ruby/compat/ripper/filter.rb +70 -0
  134. data/lib/langscan/ruby/compat/ripper/lexer.rb +179 -0
  135. data/lib/langscan/ruby/compat/ripper/sexp.rb +100 -0
  136. data/lib/langscan/scheme.rb +160 -0
  137. data/lib/langscan/sh.rb +116 -0
  138. data/lib/langscan/text.rb +37 -0
  139. data/metaconfig +2 -0
  140. data/script/console +10 -0
  141. data/script/destroy +14 -0
  142. data/script/generate +14 -0
  143. data/script/makemanifest.rb +21 -0
  144. data/setup.rb +1604 -0
  145. data/tasks/extconf.rake +13 -0
  146. data/tasks/extconf/langscan.rake +42 -0
  147. data/test/langscan/brainfuck/test/test_scan.rb +55 -0
  148. data/test/langscan/c/test/test_scan.rb +216 -0
  149. data/test/langscan/c/test/test_token.rb +41 -0
  150. data/test/langscan/csharp/test/test_scan.rb +157 -0
  151. data/test/langscan/css/test/test_css.rb +79 -0
  152. data/test/langscan/d/test/test_scan.rb +233 -0
  153. data/test/langscan/d/test/test_token.rb +205 -0
  154. data/test/langscan/eiffel/test/test_eiffel.rb +95 -0
  155. data/test/langscan/elisp/test/test_elisp.rb +177 -0
  156. data/test/langscan/io/test/test_io.rb +79 -0
  157. data/test/langscan/java/test/test_java.rb +74 -0
  158. data/test/langscan/javascript/test/test_javascript.rb +39 -0
  159. data/test/langscan/lua/test/test_lua.rb +69 -0
  160. data/test/langscan/ocaml/test/test_ocaml.rb +161 -0
  161. data/test/langscan/php/test/test_scan.rb +138 -0
  162. data/test/langscan/python/test/test_scan.rb +105 -0
  163. data/test/langscan/rpmspec/test/test_rpmspec.rb +51 -0
  164. data/test/langscan/ruby/test/test_scan.rb +71 -0
  165. data/test/langscan/scheme/test/test_scan.rb +198 -0
  166. data/test/test_helper.rb +7 -0
  167. data/test/test_langscan.rb +123 -0
  168. metadata +296 -0
@@ -0,0 +1,70 @@
1
+ #
2
+ # ripper/filter.rb
3
+ #
4
+ # Copyright (C) 2004 Minero Aoki
5
+ #
6
+ # This program is free software.
7
+ # You can distribute and/or modify this program under the Ruby License.
8
+ # For details of Ruby License, see ruby/COPYING.
9
+ #
10
+
11
+ require 'langscan/ruby/compat/ripper/lexer'
12
+
13
+ class Ripper
14
+
15
+ # This class handles only scanner events,
16
+ # and they are dispatched in the `right' order (same with input).
17
+ class Filter
18
+
19
+ def initialize(src, filename = '-', lineno = 1)
20
+ @__lexer = Lexer.new(src, filename, lineno)
21
+ @__line = nil
22
+ @__col = nil
23
+ end
24
+
25
+ # The file name of the input.
26
+ def filename
27
+ @__lexer.filename
28
+ end
29
+
30
+ # The line number of the current token.
31
+ # This value starts from 1.
32
+ # This method is valid only in event handlers.
33
+ def lineno
34
+ @__line
35
+ end
36
+
37
+ # The column number of the current token.
38
+ # This value starts from 0.
39
+ # This method is valid only in event handlers.
40
+ def column
41
+ @__col
42
+ end
43
+
44
+ # Starts parsing. _init_ is a data accumulator.
45
+ # It is passed to the next event handler (as of Enumerable#inject).
46
+ def parse(init = nil)
47
+ data = init
48
+ @__lexer.lex.each do |pos, event, tok|
49
+ @__line, @__col = *pos
50
+ data = if respond_to?(event, true)
51
+ then __send__(event, tok, data)
52
+ else on_default(event, tok, data)
53
+ end
54
+ end
55
+ data
56
+ end
57
+
58
+ private
59
+
60
+ # This method is called when some event handler have not defined.
61
+ # _event_ is :on_XXX, _token_ is scanned token, _data_ is a data
62
+ # accumulator. The return value of this method is passed to the
63
+ # next event handler (as of Enumerable#inject).
64
+ def on_default(event, token, data)
65
+ data
66
+ end
67
+
68
+ end
69
+
70
+ end
@@ -0,0 +1,179 @@
1
+ #
2
+ # ripper/lexer.rb
3
+ #
4
+ # Copyright (C) 2004,2005 Minero Aoki
5
+ #
6
+ # This program is free software.
7
+ # You can distribute and/or modify this program under the Ruby License.
8
+ # For details of Ruby License, see ruby/COPYING.
9
+ #
10
+
11
+ require 'langscan/ruby/compat/ripper/core'
12
+
13
+ class Ripper
14
+
15
+ # Tokenizes Ruby program and returns an Array of String.
16
+ def Ripper.tokenize(src, filename = '-', lineno = 1)
17
+ Lexer.new(src, filename, lineno).tokenize
18
+ end
19
+
20
+ # Tokenizes Ruby program and returns an Array of Array,
21
+ # which is formatted like [[lineno, column], type, token].
22
+ #
23
+ # require 'ripper'
24
+ # require 'pp'
25
+ #
26
+ # p Ripper.scan("def m(a) nil end")
27
+ # #=> [[[1, 0], :on_kw, "def"],
28
+ # [[1, 3], :on_sp, " " ],
29
+ # [[1, 4], :on_ident, "m" ],
30
+ # [[1, 5], :on_lparen, "(" ],
31
+ # [[1, 6], :on_ident, "a" ],
32
+ # [[1, 7], :on_rparen, ")" ],
33
+ # [[1, 8], :on_sp, " " ],
34
+ # [[1, 9], :on_kw, "nil"],
35
+ # [[1, 12], :on_sp, " " ],
36
+ # [[1, 13], :on_kw, "end"]]
37
+ #
38
+ def Ripper.lex(src, filename = '-', lineno = 1)
39
+ Lexer.new(src, filename, lineno).lex
40
+ end
41
+
42
+ class Lexer < ::Ripper #:nodoc: internal use only
43
+ def tokenize
44
+ lex().map {|pos, event, tok| tok }
45
+ end
46
+
47
+ def lex
48
+ parse().sort_by {|pos, event, tok| pos }
49
+ end
50
+
51
+ def parse
52
+ @buf = []
53
+ super
54
+ @buf
55
+ end
56
+
57
+ private
58
+
59
+ SCANNER_EVENTS.each do |event|
60
+ module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1)
61
+ def on_#{event}(tok)
62
+ @buf.push [[lineno(), column()], :on_#{event}, tok]
63
+ end
64
+ End
65
+ end
66
+ end
67
+
68
+ # [EXPERIMENTAL]
69
+ # Parses +src+ and return a string which was matched to +pattern+.
70
+ # +pattern+ should be described as Regexp.
71
+ #
72
+ # require 'ripper'
73
+ #
74
+ # p Ripper.slice('def m(a) nil end', 'ident') #=> "m"
75
+ # p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+') #=> "m(a)"
76
+ # p Ripper.slice("<<EOS\nstring\nEOS",
77
+ # 'heredoc_beg nl $(tstring_content*) heredoc_end', 1)
78
+ # #=> "string\n"
79
+ #
80
+ def Ripper.slice(src, pattern, n = 0)
81
+ if m = token_match(src, pattern)
82
+ then m.string(n)
83
+ else nil
84
+ end
85
+ end
86
+
87
+ def Ripper.token_match(src, pattern) #:nodoc:
88
+ TokenPattern.compile(pattern).match(src)
89
+ end
90
+
91
+ class TokenPattern #:nodoc:
92
+
93
+ class Error < ::StandardError; end
94
+ class CompileError < Error; end
95
+ class MatchError < Error; end
96
+
97
+ class << self
98
+ alias compile new
99
+ end
100
+
101
+ def initialize(pattern)
102
+ @source = pattern
103
+ @re = compile(pattern)
104
+ end
105
+
106
+ def match(str)
107
+ match_list(::Ripper.lex(str))
108
+ end
109
+
110
+ def match_list(tokens)
111
+ if m = @re.match(map_tokens(tokens))
112
+ then MatchData.new(tokens, m)
113
+ else nil
114
+ end
115
+ end
116
+
117
+ private
118
+
119
+ def compile(pattern)
120
+ if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern)
121
+ raise CompileError, "invalid char in pattern: #{m[0].inspect}"
122
+ end
123
+ buf = ''
124
+ pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok|
125
+ case tok
126
+ when /\w/
127
+ buf.concat map_token(tok)
128
+ when '$('
129
+ buf.concat '('
130
+ when '('
131
+ buf.concat '(?:'
132
+ when /[?*\[\])\.]/
133
+ buf.concat tok
134
+ else
135
+ raise 'must not happen'
136
+ end
137
+ end
138
+ Regexp.compile(buf)
139
+ rescue RegexpError => err
140
+ raise CompileError, err.message
141
+ end
142
+
143
+ def map_tokens(tokens)
144
+ tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join
145
+ end
146
+
147
+ MAP = {}
148
+ seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
149
+ SCANNER_EVENT_TABLE.each do |ev, |
150
+ raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty?
151
+ MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift
152
+ end
153
+
154
+ def map_token(tok)
155
+ MAP[tok] or raise CompileError, "unknown token: #{tok}"
156
+ end
157
+
158
+ class MatchData
159
+ def initialize(tokens, match)
160
+ @tokens = tokens
161
+ @match = match
162
+ end
163
+
164
+ def string(n = 0)
165
+ return nil unless @match
166
+ match(n).join
167
+ end
168
+
169
+ private
170
+
171
+ def match(n = 0)
172
+ return [] unless @match
173
+ @tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str }
174
+ end
175
+ end
176
+
177
+ end
178
+
179
+ end
@@ -0,0 +1,100 @@
1
+ #
2
+ # ripper/sexp.rb
3
+ #
4
+ # Copyright (C) 2004,2005 Minero Aoki
5
+ #
6
+ # This program is free software.
7
+ # You can distribute and/or modify this program under the Ruby License.
8
+ # For details of Ruby License, see ruby/COPYING.
9
+ #
10
+
11
+ require 'langscan/ruby/compat/ripper/core'
12
+
13
+ class Ripper
14
+
15
+ # [EXPERIMENTAL]
16
+ # Parses +src+ and create S-exp tree.
17
+ # This method is for mainly developper use.
18
+ #
19
+ # require 'ripper'
20
+ # require 'pp
21
+ #
22
+ # pp Ripper.sexp("def m(a) nil end")
23
+ # #=> [:program,
24
+ # [:stmts_add,
25
+ # [:stmts_new],
26
+ # [:def,
27
+ # [:@ident, "m", [1, 4]],
28
+ # [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
29
+ # [:bodystmt,
30
+ # [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
31
+ # nil,
32
+ # nil,
33
+ # nil]]]]
34
+ #
35
+ def Ripper.sexp(src, filename = '-', lineno = 1)
36
+ SexpBuilderPP.new(src, filename, lineno).parse
37
+ end
38
+
39
+ def Ripper.sexp_raw(src, filename = '-', lineno = 1)
40
+ SexpBuilder.new(src, filename, lineno).parse
41
+ end
42
+
43
+ class SexpBuilderPP < ::Ripper #:nodoc:
44
+ private
45
+
46
+ PARSER_EVENTS.each do |event|
47
+ case event.to_s
48
+ when /_new\z/
49
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
50
+ def on_#{event}(*args)
51
+ []
52
+ end
53
+ End
54
+ when /_add\z/
55
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
56
+ def on_#{event}(list, item)
57
+ list.push item
58
+ list
59
+ end
60
+ End
61
+ else
62
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
63
+ def on_#{event}(*args)
64
+ [:#{event}, *args]
65
+ end
66
+ End
67
+ end
68
+ end
69
+
70
+ SCANNER_EVENTS.each do |event|
71
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
72
+ def on_#{event}(tok)
73
+ [:@#{event}, tok, [lineno(), column()]]
74
+ end
75
+ End
76
+ end
77
+ end
78
+
79
+ class SexpBuilder < ::Ripper #:nodoc:
80
+ private
81
+
82
+ PARSER_EVENTS.each do |event|
83
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
84
+ def on_#{event}(*args)
85
+ args.unshift :#{event}
86
+ args
87
+ end
88
+ End
89
+ end
90
+
91
+ SCANNER_EVENTS.each do |event|
92
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
93
+ def on_#{event}(tok)
94
+ [:@#{event}, tok, [lineno(), column()]]
95
+ end
96
+ End
97
+ end
98
+ end
99
+
100
+ end
@@ -0,0 +1,160 @@
1
+ #
2
+ # scheme.rb - a Scheme module of LangScan
3
+ #
4
+ # Copyright (C) 2005 Kenichi Ishibashi <bashi at dream.ie.ariake-nct.ac.jp>
5
+ # All rights reserved.
6
+ # This is free software with ABSOLUTELY NO WARRANTY.
7
+ #
8
+ # You can redistribute it and/or modify it under the terms of
9
+ # the GNU General Public License version 2.
10
+ #
11
+ require 'langscan/scheme/scheme'
12
+ require 'langscan/_common'
13
+ require 'langscan/_pairmatcher'
14
+
15
+ class Struct::LangScanPair
16
+ def each_outer
17
+ ret = self
18
+ while o = ret.outer
19
+ yield o
20
+ end
21
+ end
22
+ end
23
+
24
+ module LangScan
25
+ module Scheme
26
+ module_function
27
+ def name
28
+ "Scheme"
29
+ end
30
+
31
+
32
+ def abbrev
33
+ "scheme"
34
+ end
35
+
36
+ def extnames
37
+ [".scm"]
38
+ end
39
+
40
+ # LangScan::Scheme.scan iterates over Scheme program.
41
+ # It yields for each Fragment.
42
+ def scan(input, &block)
43
+ sorter = PairMatcher.fragmentsorter(block)
44
+ scan_unsorted(input, &sorter)
45
+ end
46
+
47
+ def scan_unsorted(input, &block)
48
+ pm = LangScan::PairMatcher.new(2,2,2,2)
49
+ pm.define_intertoken_fragment :space, nil
50
+ pm.define_intertoken_fragment :comment, nil
51
+ pm.define_pair :paren, :punct, "(", :punct, ")"
52
+ pm.define_pair :vector, :punct, "#(", :punct, ")"
53
+ reporter = lambda {|f|
54
+ if (f.type == :ident || f.type == :funcall) && KeywordsHash[f.text]
55
+ f.type = :keyword
56
+ end
57
+ if f.type == :number
58
+ f.type = if f.text.include?("i")
59
+ :imaginary
60
+ elsif f.text =~ /[.\/]|[0-9]#*[esfdl][0-9\-+]/
61
+ :floating
62
+ else
63
+ :integer
64
+ end
65
+ end
66
+ if f.type == :quote_chars
67
+ f.type = :punct
68
+ end
69
+ yield f
70
+ }
71
+ pm.parse(LangScan::Scheme::Tokenizer.new(input), reporter) {|list|
72
+ if list.around_open(1).type == :ident
73
+ list.around_open(1).type = case
74
+ when fundef_list?(list)
75
+ :fundef
76
+ when funcall_list?(list)
77
+ :funcall
78
+ else
79
+ :ident
80
+ end
81
+ end
82
+ }
83
+ end
84
+
85
+ def fundef_list?(list)
86
+ if list.before_open_length >= 1 && list.around_open(-1).text == "define"
87
+ return true
88
+ end
89
+ return false
90
+ end
91
+
92
+ def funcall_list?(list)
93
+ if list.before_open_length == 0
94
+ return true
95
+ end
96
+ if NotFuncallWordsHash[list.around_open(-1).text]
97
+ return false
98
+ end
99
+ if quote_list?(list)
100
+ return false
101
+ end
102
+ outer = list.outer
103
+ second_outer = outer.outer unless outer == nil
104
+ if second_outer and NotFuncall2ndOuterWordsHash[second_outer.around_open(1).text]
105
+ if NotFuncall2ndOuterWordsHash[outer.around_open(-1).text]
106
+ return false
107
+ end
108
+ end
109
+ return true
110
+ end
111
+
112
+ def quote_nestlevel(str)
113
+ str.count("`") - str.count(",")
114
+ end
115
+
116
+ def quote_list?(list)
117
+ l = list
118
+ nest = 0
119
+ while l
120
+ if l.before_open_length >= 1
121
+ before = l.around_open(-1)
122
+ if before.type == :quote_chars
123
+ return true if before.text.include?("'")
124
+ nest = nest + quote_nestlevel(before.text)
125
+ end
126
+ end
127
+ if l.after_open_length >= 1
128
+ after = l.around_open(1)
129
+ if after.text == 'quote'
130
+ return true
131
+ elsif after.text == 'quasiquote'
132
+ nest = nest + 1
133
+ elsif after.text == 'unquote'
134
+ nest = nest - 1
135
+ end
136
+ end
137
+ l = l.outer
138
+ end
139
+ return nest > 0
140
+ end
141
+
142
+ Keywords = %w(
143
+ else => define unquote unquote-splicing quote lambda if
144
+ set! begin cond and or case let let* letrec do delay quasiquote
145
+ syntax-rules define-syntax
146
+ )
147
+ KeywordsHash = {}
148
+ Keywords.each {|k| KeywordsHash[k] = k }
149
+
150
+ NotFuncallWordsHash = {"lambda" => "lambda"}
151
+
152
+ NotFuncall2ndOuterWords = %w(
153
+ let let* letrec let-syntax letrec-syntax do
154
+ )
155
+ NotFuncall2ndOuterWordsHash = {}
156
+ NotFuncall2ndOuterWords.each {|k| NotFuncall2ndOuterWordsHash[k] = k }
157
+
158
+ LangScan.register(self)
159
+ end
160
+ end