langscan 1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (168) hide show
  1. data/AUTHORS.txt +19 -0
  2. data/History.txt +126 -0
  3. data/Manifest.txt +167 -0
  4. data/README.rdoc +89 -0
  5. data/Rakefile +40 -0
  6. data/ext/langscan/_make_c.rb +20 -0
  7. data/ext/langscan/_make_h.rb +30 -0
  8. data/ext/langscan/_template.c +134 -0
  9. data/ext/langscan/_template.h +53 -0
  10. data/ext/langscan/c/c/Makefile +157 -0
  11. data/ext/langscan/c/c/c.c +134 -0
  12. data/ext/langscan/c/c/c.h +66 -0
  13. data/ext/langscan/c/c/ctok.c +4622 -0
  14. data/ext/langscan/c/c/ctok.l +212 -0
  15. data/ext/langscan/c/c/extconf.rb +3 -0
  16. data/ext/langscan/c/c/modulename.txt +1 -0
  17. data/ext/langscan/c/c/tokenlist.txt +13 -0
  18. data/ext/langscan/csharp/csharp/Makefile +157 -0
  19. data/ext/langscan/csharp/csharp/csharp.c +134 -0
  20. data/ext/langscan/csharp/csharp/csharp.h +65 -0
  21. data/ext/langscan/csharp/csharp/csharptok.c +2965 -0
  22. data/ext/langscan/csharp/csharp/csharptok.l +200 -0
  23. data/ext/langscan/csharp/csharp/extconf.rb +3 -0
  24. data/ext/langscan/csharp/csharp/modulename.txt +1 -0
  25. data/ext/langscan/csharp/csharp/tokenlist.txt +12 -0
  26. data/ext/langscan/d/d/Makefile +157 -0
  27. data/ext/langscan/d/d/d.c +134 -0
  28. data/ext/langscan/d/d/d.h +64 -0
  29. data/ext/langscan/d/d/dtok.c +5461 -0
  30. data/ext/langscan/d/d/dtok.l +282 -0
  31. data/ext/langscan/d/d/extconf.rb +3 -0
  32. data/ext/langscan/d/d/modulename.txt +1 -0
  33. data/ext/langscan/d/d/tokenlist.txt +11 -0
  34. data/ext/langscan/elisp/elisp/Makefile +157 -0
  35. data/ext/langscan/elisp/elisp/elisp.c +134 -0
  36. data/ext/langscan/elisp/elisp/elisp.h +62 -0
  37. data/ext/langscan/elisp/elisp/elisptok.c +2101 -0
  38. data/ext/langscan/elisp/elisp/elisptok.l +151 -0
  39. data/ext/langscan/elisp/elisp/extconf.rb +3 -0
  40. data/ext/langscan/elisp/elisp/modulename.txt +1 -0
  41. data/ext/langscan/elisp/elisp/tokenlist.txt +9 -0
  42. data/ext/langscan/java/java/Makefile +157 -0
  43. data/ext/langscan/java/java/extconf.rb +3 -0
  44. data/ext/langscan/java/java/java.c +134 -0
  45. data/ext/langscan/java/java/java.h +64 -0
  46. data/ext/langscan/java/java/javatok.c +2090 -0
  47. data/ext/langscan/java/java/javatok.l +155 -0
  48. data/ext/langscan/java/java/modulename.txt +1 -0
  49. data/ext/langscan/java/java/tokenlist.txt +11 -0
  50. data/ext/langscan/javascript/javascript/Makefile +157 -0
  51. data/ext/langscan/javascript/javascript/extconf.rb +3 -0
  52. data/ext/langscan/javascript/javascript/javascript.c +134 -0
  53. data/ext/langscan/javascript/javascript/javascript.h +63 -0
  54. data/ext/langscan/javascript/javascript/javascripttok.c +2051 -0
  55. data/ext/langscan/javascript/javascript/javascripttok.l +147 -0
  56. data/ext/langscan/javascript/javascript/modulename.txt +1 -0
  57. data/ext/langscan/javascript/javascript/tokenlist.txt +10 -0
  58. data/ext/langscan/pairmatcher/pairmatcher/Makefile +157 -0
  59. data/ext/langscan/pairmatcher/pairmatcher/extconf.rb +3 -0
  60. data/ext/langscan/pairmatcher/pairmatcher/pairmatcher.c +890 -0
  61. data/ext/langscan/php/php/Makefile +157 -0
  62. data/ext/langscan/php/php/extconf.rb +3 -0
  63. data/ext/langscan/php/php/modulename.txt +1 -0
  64. data/ext/langscan/php/php/php.c +134 -0
  65. data/ext/langscan/php/php/php.h +64 -0
  66. data/ext/langscan/php/php/phptok.c +2406 -0
  67. data/ext/langscan/php/php/phptok.l +212 -0
  68. data/ext/langscan/php/php/tokenlist.txt +11 -0
  69. data/ext/langscan/post-distclean.rb +21 -0
  70. data/ext/langscan/pre-config.rb +57 -0
  71. data/ext/langscan/python/python/Makefile +157 -0
  72. data/ext/langscan/python/python/extconf.rb +3 -0
  73. data/ext/langscan/python/python/modulename.txt +1 -0
  74. data/ext/langscan/python/python/python.c +134 -0
  75. data/ext/langscan/python/python/python.h +61 -0
  76. data/ext/langscan/python/python/pythontok.c +2102 -0
  77. data/ext/langscan/python/python/pythontok.l +155 -0
  78. data/ext/langscan/python/python/tokenlist.txt +8 -0
  79. data/ext/langscan/ruby/compat/ripper/Makefile +158 -0
  80. data/ext/langscan/ruby/compat/ripper/depend +1 -0
  81. data/ext/langscan/ruby/compat/ripper/extconf.rb +4 -0
  82. data/ext/langscan/ruby/compat/ripper/include/eventids1.c +251 -0
  83. data/ext/langscan/ruby/compat/ripper/include/eventids2.c +277 -0
  84. data/ext/langscan/ruby/compat/ripper/include/lex.c +138 -0
  85. data/ext/langscan/ruby/compat/ripper/ripper.c +14420 -0
  86. data/ext/langscan/scheme/scheme/Makefile +157 -0
  87. data/ext/langscan/scheme/scheme/extconf.rb +3 -0
  88. data/ext/langscan/scheme/scheme/modulename.txt +1 -0
  89. data/ext/langscan/scheme/scheme/scheme.c +134 -0
  90. data/ext/langscan/scheme/scheme/scheme.h +60 -0
  91. data/ext/langscan/scheme/scheme/schemetok.c +2447 -0
  92. data/ext/langscan/scheme/scheme/schemetok.l +177 -0
  93. data/ext/langscan/scheme/scheme/tokenlist.txt +7 -0
  94. data/ext/langscan/sh/sh/Makefile +157 -0
  95. data/ext/langscan/sh/sh/extconf.rb +3 -0
  96. data/ext/langscan/sh/sh/modulename.txt +1 -0
  97. data/ext/langscan/sh/sh/sh.c +134 -0
  98. data/ext/langscan/sh/sh/sh.h +61 -0
  99. data/ext/langscan/sh/sh/shtok.c +2470 -0
  100. data/ext/langscan/sh/sh/shtok.l +325 -0
  101. data/ext/langscan/sh/sh/tokenlist.txt +8 -0
  102. data/lib/langscan.rb +124 -0
  103. data/lib/langscan/_common.rb +50 -0
  104. data/lib/langscan/_easyscanner.rb +78 -0
  105. data/lib/langscan/_pairmatcher.rb +46 -0
  106. data/lib/langscan/_type.rb +125 -0
  107. data/lib/langscan/autoconf.rb +51 -0
  108. data/lib/langscan/automake.rb +51 -0
  109. data/lib/langscan/brainfuck.rb +48 -0
  110. data/lib/langscan/c.rb +144 -0
  111. data/lib/langscan/csharp.rb +101 -0
  112. data/lib/langscan/css.rb +109 -0
  113. data/lib/langscan/d.rb +201 -0
  114. data/lib/langscan/eiffel.rb +167 -0
  115. data/lib/langscan/elisp.rb +132 -0
  116. data/lib/langscan/io.rb +84 -0
  117. data/lib/langscan/java.rb +95 -0
  118. data/lib/langscan/javascript.rb +97 -0
  119. data/lib/langscan/lua.rb +116 -0
  120. data/lib/langscan/ocaml.rb +298 -0
  121. data/lib/langscan/ocaml/camlexer.ml +28 -0
  122. data/lib/langscan/ocaml/lexer.mll +230 -0
  123. data/lib/langscan/ocaml/types.ml +36 -0
  124. data/lib/langscan/perl.rb +87 -0
  125. data/lib/langscan/perl/tokenizer.pl +231 -0
  126. data/lib/langscan/php.rb +80 -0
  127. data/lib/langscan/python.rb +101 -0
  128. data/lib/langscan/rpmspec.rb +71 -0
  129. data/lib/langscan/ruby.rb +164 -0
  130. data/lib/langscan/ruby/compat/README +5 -0
  131. data/lib/langscan/ruby/compat/ripper.rb +4 -0
  132. data/lib/langscan/ruby/compat/ripper/core.rb +918 -0
  133. data/lib/langscan/ruby/compat/ripper/filter.rb +70 -0
  134. data/lib/langscan/ruby/compat/ripper/lexer.rb +179 -0
  135. data/lib/langscan/ruby/compat/ripper/sexp.rb +100 -0
  136. data/lib/langscan/scheme.rb +160 -0
  137. data/lib/langscan/sh.rb +116 -0
  138. data/lib/langscan/text.rb +37 -0
  139. data/metaconfig +2 -0
  140. data/script/console +10 -0
  141. data/script/destroy +14 -0
  142. data/script/generate +14 -0
  143. data/script/makemanifest.rb +21 -0
  144. data/setup.rb +1604 -0
  145. data/tasks/extconf.rake +13 -0
  146. data/tasks/extconf/langscan.rake +42 -0
  147. data/test/langscan/brainfuck/test/test_scan.rb +55 -0
  148. data/test/langscan/c/test/test_scan.rb +216 -0
  149. data/test/langscan/c/test/test_token.rb +41 -0
  150. data/test/langscan/csharp/test/test_scan.rb +157 -0
  151. data/test/langscan/css/test/test_css.rb +79 -0
  152. data/test/langscan/d/test/test_scan.rb +233 -0
  153. data/test/langscan/d/test/test_token.rb +205 -0
  154. data/test/langscan/eiffel/test/test_eiffel.rb +95 -0
  155. data/test/langscan/elisp/test/test_elisp.rb +177 -0
  156. data/test/langscan/io/test/test_io.rb +79 -0
  157. data/test/langscan/java/test/test_java.rb +74 -0
  158. data/test/langscan/javascript/test/test_javascript.rb +39 -0
  159. data/test/langscan/lua/test/test_lua.rb +69 -0
  160. data/test/langscan/ocaml/test/test_ocaml.rb +161 -0
  161. data/test/langscan/php/test/test_scan.rb +138 -0
  162. data/test/langscan/python/test/test_scan.rb +105 -0
  163. data/test/langscan/rpmspec/test/test_rpmspec.rb +51 -0
  164. data/test/langscan/ruby/test/test_scan.rb +71 -0
  165. data/test/langscan/scheme/test/test_scan.rb +198 -0
  166. data/test/test_helper.rb +7 -0
  167. data/test/test_langscan.rb +123 -0
  168. metadata +296 -0
@@ -0,0 +1,79 @@
1
+ $LOAD_PATH << "../../.." unless $LOAD_PATH.include?("../../..")
2
+ require 'test/unit'
3
+ require 'langscan/css'
4
+
5
+ class TestCSS < Test::Unit::TestCase
6
+ def assert_fragments(expected, code)
7
+ result = []
8
+ LangScan::CSS.scan(code) {|f| result.push(f) }
9
+ fragments = []
10
+ result.each {|fragment|
11
+ type, text = fragment.to_a
12
+ unless type == :punct
13
+ fragments << [type, text]
14
+ end
15
+ }
16
+ assert_equal(expected, fragments)
17
+ end
18
+
19
+ def test_sample_css
20
+ code = <<"End"
21
+ @import "another.css";
22
+
23
+ h1 {
24
+ border: 1px solid #ccc;
25
+ }
26
+
27
+ div.day {
28
+ font: medium bold sans-serif;
29
+ }
30
+ End
31
+ expected = [[:keyword, "@import"],
32
+ [:string, "\"another.css\""],
33
+ [:fundef, "h1"],
34
+ [:keyword, "border"],
35
+ [:ident, "1px"],
36
+ [:ident, "solid"],
37
+ [:ident, "ccc"],
38
+ [:fundef, "div"],
39
+ [:fundef, "day"],
40
+ [:keyword, "font"],
41
+ [:ident, "medium"],
42
+ [:ident, "bold"],
43
+ [:ident, "sans-serif"]]
44
+ assert_fragments(expected, code)
45
+ end
46
+
47
+ def test_comment
48
+ assert_fragments([[:comment, "/*c*/"]], "/*c*/")
49
+ end
50
+
51
+ def test_string
52
+ assert_fragments([[:keyword, "url"], [:string, "(a)"]], "url(a)")
53
+ end
54
+
55
+ def test_keyword
56
+ assert_fragments([[:keyword, "@import"], [:string, "\"a.css\""]],
57
+ "@import \"a.css\";")
58
+ assert_fragments([[:keyword, "!important"]], "!important")
59
+ assert_fragments([[:keyword, "! important"]], "! important")
60
+ end
61
+
62
+ def test_selector
63
+ assert_fragments([[:fundef, "p"]], "p{}")
64
+ assert_fragments([[:fundef, "c"]], ".c{}")
65
+ assert_fragments([[:fundef, "i"]], "#i{}")
66
+ assert_fragments([[:fundef, "p"], [:fundef, "c"]], "p.c{}")
67
+ assert_fragments([[:fundef, "p"], [:fundef, "b"]], "p b{}")
68
+ assert_fragments([[:fundef, "h1"], [:fundef, "h2"]], "h1,h2{}")
69
+ assert_fragments([[:fundef, "p"], [:fundef, "b"]], "p>b{}")
70
+ assert_fragments([[:fundef, "a"], [:fundef, "active"]], "a:active{}")
71
+ end
72
+
73
+ def test_property
74
+ assert_fragments([[:fundef, "p"], [:keyword, "b"], [:ident, "a"]],
75
+ "p{b:a}")
76
+ assert_fragments([[:fundef, "p"], [:keyword, "border"], [:ident, "0"]],
77
+ "p{border:0;}")
78
+ end
79
+ end
@@ -0,0 +1,233 @@
1
+ require 'test/unit'
2
+ require 'langscan/d'
3
+
4
+ class TestScanD < Test::Unit::TestCase
5
+ def assert_yield(recv, meth, *args)
6
+ yielded = false
7
+ recv.__send__(meth, *args) {|*block_args|
8
+ yielded = true
9
+ yield(*block_args)
10
+ }
11
+ assert(yielded, "block not yielded")
12
+ end
13
+
14
+ def assert_not_yield(recv, meth, *args)
15
+ yielded = false
16
+ recv.__send__(meth, *args) {|*block_args|
17
+ assert(false, "block yielded")
18
+ }
19
+ end
20
+
21
+ def test_token_position
22
+ assert_yield(LangScan::D, :scan, "a(1,2,3)") {|f|
23
+ next if f.type != :funcall
24
+ assert_equal(0, f.beg_byteno)
25
+ #assert_equal(1, f.end_byteno)
26
+ }
27
+ end
28
+
29
+ def assert_yield_any(recv, meth, *args)
30
+ success = false
31
+ recv.__send__(meth, *args) {|*block_args|
32
+ if yield(*block_args)
33
+ assert(true)
34
+ success = true
35
+ break
36
+ end
37
+ }
38
+ assert(false, "no expected yields") unless success
39
+ end
40
+
41
+ def assert_yield_all(recv, meth, *args)
42
+ recv.__send__(meth, *args) {|*block_args|
43
+ if !yield(*block_args)
44
+ assert(false, "unexpected yields")
45
+ return
46
+ end
47
+ }
48
+ assert(true)
49
+ end
50
+
51
+ def test_funtype
52
+ assert_yield_all(LangScan::D, :scan, "int fun(type (*)())") {|f|
53
+ !(f.type == :fundef || f.type == :funcall || f.type == :fundecl) ||
54
+ f.text == 'fun'
55
+ }
56
+ end
57
+
58
+ def test_decl
59
+ assert_yield_any(LangScan::D, :scan, "int f();") {|f|
60
+ f.type == :fundecl && f.text == 'f'
61
+ }
62
+ end
63
+
64
+ def test_preproc_defined
65
+ assert_yield_all(LangScan::D, :scan, "#if defined(MACRO)\n") {|f|
66
+ !(f.type == :funcall && f.text == 'defined')
67
+ }
68
+ end
69
+
70
+ def test_fragment_and_function
71
+ regions = {}
72
+ LangScan::D.scan("a()") {|t|
73
+ f = t
74
+ f = f.name_token if f.respond_to? :name_token
75
+ r = f.beg_byteno
76
+ assert(!regions.include?(r), "duplicate token: #{regions[r].inspect} and #{t.inspect}")
77
+ regions[r] = t
78
+ }
79
+ end
80
+
81
+ def test_funcdef
82
+ [
83
+ "int fun() {}",
84
+ "int fun(void) {}",
85
+ "int fun(int arg) {}",
86
+ ].each {|src|
87
+ result = false
88
+ assert_yield(LangScan::D, :scan, src) {|f|
89
+ if f.type == :fundef && f.text == 'fun'
90
+ result = true
91
+ end
92
+ }
93
+ assert(result, src)
94
+ }
95
+ end
96
+
97
+ def test_extern_c
98
+ src = 'extern "C" { void fun(); }'
99
+ assert_yield_any(LangScan::D, :scan, src) {|f|
100
+ f.type == :fundecl && f.text == 'fun'
101
+ }
102
+ end
103
+
104
+ def test_keyword
105
+ assert_yield_all(LangScan::D, :scan, "int()") {|f|
106
+ f.type != :funcall
107
+ }
108
+ end
109
+
110
+ def test_toplevel_comma
111
+ assert_yield(LangScan::D, :scan, ",") {|f|
112
+ assert_equal(LangScan::Fragment, f.class)
113
+ assert_equal(:punct, f.type)
114
+ assert_equal(",", f.text)
115
+ }
116
+ end
117
+
118
+ def test_funcall
119
+ result = []
120
+ LangScan::D.scan("f(){g();}") {|f|
121
+ next unless f.type == :fundef || f.type == :funcall
122
+ result << [f.type, f.text]
123
+ }
124
+ assert_equal([[:fundef, 'f'], [:funcall, 'g']], result)
125
+ end
126
+
127
+ # C++
128
+
129
+ def assert_fragment_type(type, text, src)
130
+ found = false
131
+ LangScan::D.scan(src) {|f|
132
+ if f.text == text
133
+ if found
134
+ raise "token #{text} occurred twice"
135
+ else
136
+ found = true
137
+ assert_equal(type, f.type, "fragment type of #{text}")
138
+ end
139
+ end
140
+ }
141
+ unless found
142
+ raise "token #{text} not found"
143
+ end
144
+ end
145
+
146
+ def test_class
147
+ assert_fragment_type(:classdef, 'c', "class c {};")
148
+ assert_fragment_type(:classdef, 'c1', "class c1 : c2 {};")
149
+ assert_fragment_type(:classref, 'c2', "class c1 : c2 {};")
150
+ end
151
+
152
+ def test_struct
153
+ assert_fragment_type(:classdef, 'c', "struct c {};")
154
+ assert_fragment_type(:classdef, 'c1', "struct c1 : c2 {};")
155
+ assert_fragment_type(:classref, 'c2', "struct c1 : c2 {};")
156
+ end
157
+
158
+ def check_scan(src)
159
+ LangScan::D.scan(src) {|f|
160
+ assert_equal(f.text, src[f.beg_byteno...f.end_byteno])
161
+ }
162
+ end
163
+
164
+ def test_sharp_in_non_initial_state
165
+ check_scan("struct x\n#;")
166
+ end
167
+
168
+ def test_fundef_returns_user_defined_type
169
+ assert_fragment_type(:fundef, 'foo', 'VALUE foo() {}')
170
+ end
171
+
172
+ def test_classdescls
173
+ assert_fragment_type(:classdef, 'Foo', 'class Foo { int a; }')
174
+ assert_fragment_type(:classdef, 'Foo', 'class Foo { int a; } var;')
175
+ assert_fragment_type(:classdef, 'Foo', 'abstract class Foo : A {}')
176
+ assert_fragment_type(:classdef, 'Foo', 'class Foo : A, B {}')
177
+ assert_fragment_type(:classdef, 'Foo', 'class Foo { class Inner {} }')
178
+ assert_fragment_type(:classdef, 'Foo', 'interface Foo {}')
179
+ assert_fragment_type(:classdef, 'Foo', 'struct Foo {}')
180
+ assert_fragment_type(:classdef, 'Foo', 'union Foo {}')
181
+ assert_fragment_type(:classdef, 'Foo', 'enum Foo {}')
182
+ end
183
+
184
+ def test_fundefs
185
+ assert_fragment_type(:fundef, 'foo', 'VALUE foo() {}')
186
+ assert_fragment_type(:fundef, 'foo', 'unittest { VALUE foo() {} }')
187
+ assert_fragment_type(:fundef, 'foo', 'version(HOGE) { VALUE foo() {} }')
188
+ assert_fragment_type(:fundef, 'foo', 'void x() { VALUE foo() {} }')
189
+ assert_fragment_type(:fundef, 'foo', 'void x(...) { VALUE foo() {} }')
190
+ end
191
+
192
+ def test_fundecls
193
+ assert_fragment_type(:fundecl, 'foo', 'VALUE foo();')
194
+ assert_fragment_type(:fundecl, 'foo', 'void x() { class C { foo(); } }')
195
+ assert_fragment_type(:fundecl, 'foo', 'version(HOGE) { VALUE foo(); }')
196
+ assert_fragment_type(:fundecl, 'foo', 'template T { VALUE foo(); }')
197
+ assert_fragment_type(:fundecl, 'foo', 'template T(T) { VALUE foo(); }')
198
+ assert_fragment_type(:fundecl, 'foo', 'template T(T,T) { VALUE foo(); }')
199
+ assert_fragment_type(:fundecl, 'foo', 'template T(T,T,T) { VALUE foo(); }')
200
+ assert_fragment_type(:fundecl, 'foo', 'template T(T:char[]) { VALUE foo(); }')
201
+ assert_fragment_type(:fundecl, 'foo', 'void x(int a, int b, int c, int d, ...) { class C { foo(); } }')
202
+ end
203
+
204
+ def test_funcalls
205
+ assert_fragment_type(:funcall, 'foo', 'x(foo());')
206
+ assert_fragment_type(:funcall, 'foo', 'void x() { foo(); }')
207
+ assert_fragment_type(:funcall, 'foo', 'void x() { a.foo(); }')
208
+ assert_fragment_type(:funcall, 'foo', 'unittest { VALUE foo(); }')
209
+ assert_fragment_type(:funcall, 'foo', 'void x() body { VALUE foo(); }')
210
+ assert_fragment_type(:funcall, 'foo', 'void x() in { VALUE foo(); }')
211
+ assert_fragment_type(:funcall, 'foo', 'out { VALUE foo(); }')
212
+ assert_fragment_type(:funcall, 'foo', 'void x() {{{{VALUE foo();}}}}')
213
+ assert_fragment_type(:funcall, 'foo', 'for (int i=0; i<10; i++) { VALUE foo(); }')
214
+ assert_fragment_type(:funcall, 'foo', 'void x() { debug(D) { VALUE foo(); } }')
215
+ assert_fragment_type(:funcall, 'foo', 'void x() { with(c) { VALUE foo(); } }')
216
+ end
217
+
218
+ def test_delegates
219
+ assert_fragment_type(:keyword, 'delegate', 'void delegate() fp;')
220
+ assert_fragment_type(:keyword, 'delegate', 'return delegate void() {}')
221
+ assert_fragment_type(:ident, 'type', 'return delegate type() {}')
222
+ assert_fragment_type(:keyword, 'function', 'void function() fp;')
223
+ assert_fragment_type(:keyword, 'function', 'return function type() {}')
224
+ assert_fragment_type(:ident, 'type', 'return function type() {}')
225
+ end
226
+
227
+ def test_funcables
228
+ assert_fragment_type(:keyword, 'new', 'c = new C();')
229
+ assert_fragment_type(:keyword, 'delete', 'delete p;')
230
+ assert_fragment_type(:keyword, 'this', 'return this;')
231
+ assert_fragment_type(:keyword, '~this', '~this();')
232
+ end
233
+ end
@@ -0,0 +1,205 @@
1
+ require 'test/unit'
2
+ require 'langscan/d'
3
+
4
+ class TestTokenD < Test::Unit::TestCase
5
+ def test_d_puncts
6
+ t = LangScan::D::Tokenizer.new('~=')
7
+ type, text = t.get_token
8
+ assert_equal('~=', text)
9
+ assert_equal(:punct, type)
10
+ assert_equal(nil, t.get_token)
11
+
12
+ t = LangScan::D::Tokenizer.new('>>>=')
13
+ type, text = t.get_token
14
+ assert_equal('>>>=', text)
15
+ assert_equal(:punct, type)
16
+ assert_equal(nil, t.get_token)
17
+ end
18
+
19
+ def test_d_idents
20
+ t = LangScan::D::Tokenizer.new('~this')
21
+ type, text = t.get_token
22
+ assert_equal('~this', text)
23
+ assert_equal(:ident, type)
24
+ assert_equal(nil, t.get_token)
25
+ end
26
+
27
+ def test_d_preprocs
28
+ t = LangScan::D::Tokenizer.new('#line 6 "foo\bar"')
29
+ type, text = t.get_token
30
+ assert_equal('#line 6 "foo\bar"', text)
31
+ assert_equal(:preproc, type)
32
+ assert_equal(nil, t.get_token)
33
+
34
+ t = LangScan::D::Tokenizer.new('int #line 6 "foo\bar"')
35
+ type, text = t.get_token # int
36
+ type, text = t.get_token # space
37
+ type, text = t.get_token
38
+ assert_equal('#line 6 "foo\bar"', text)
39
+ assert_equal(:preproc, type)
40
+ assert_equal(nil, t.get_token)
41
+ end
42
+
43
+ def test_d_escape_integers
44
+ t = LangScan::D::Tokenizer.new('0XA2FlU')
45
+ type, text = t.get_token
46
+ assert_equal('0XA2FlU', text)
47
+ assert_equal(:integer, type)
48
+ assert_equal(nil, t.get_token)
49
+
50
+ t = LangScan::D::Tokenizer.new('0b100L_0127u__')
51
+ type, text = t.get_token
52
+ assert_equal('0b100L_0127u__', text)
53
+ assert_equal(:integer, type)
54
+ assert_equal(nil, t.get_token)
55
+ end
56
+
57
+ def test_d_escape_floatings
58
+ t = LangScan::D::Tokenizer.new('123_456.567_8')
59
+ type, text = t.get_token
60
+ assert_equal('123_456.567_8', text)
61
+ assert_equal(:floating, type)
62
+ assert_equal(nil, t.get_token)
63
+
64
+ t = LangScan::D::Tokenizer.new('1_2_3_4_5_6_._5e-6_fI')
65
+ type, text = t.get_token
66
+ assert_equal('1_2_3_4_5_6_._5e-6_fI', text)
67
+ assert_equal(:floating, type)
68
+ assert_equal(nil, t.get_token)
69
+
70
+ t = LangScan::D::Tokenizer.new('0x1.FFFFFFFFFFFFFp1023i')
71
+ type, text = t.get_token
72
+ assert_equal('0x1.FFFFFFFFFFFFFp1023i', text)
73
+ assert_equal(:floating, type)
74
+ assert_equal(nil, t.get_token)
75
+ end
76
+
77
+ def test_d_escape_strings
78
+ t = LangScan::D::Tokenizer.new('\?')
79
+ type, text = t.get_token
80
+ assert_equal('\?', text)
81
+ assert_equal(:string, type)
82
+ assert_equal(nil, t.get_token)
83
+
84
+ t = LangScan::D::Tokenizer.new('\200')
85
+ type, text = t.get_token
86
+ assert_equal('\200', text)
87
+ assert_equal(:string, type)
88
+ assert_equal(nil, t.get_token)
89
+
90
+ t = LangScan::D::Tokenizer.new('\UF0101234')
91
+ type, text = t.get_token
92
+ assert_equal('\UF0101234', text)
93
+ assert_equal(:string, type)
94
+ assert_equal(nil, t.get_token)
95
+
96
+ t = LangScan::D::Tokenizer.new('\&reg;')
97
+ type, text = t.get_token
98
+ assert_equal('\&reg;', text)
99
+ assert_equal(:string, type)
100
+ assert_equal(nil, t.get_token)
101
+ end
102
+
103
+ def test_d_escapes
104
+ t = LangScan::D::Tokenizer.new('"\""')
105
+ type, text = t.get_token
106
+ assert_equal('"\""', text)
107
+ assert_equal(:string, type)
108
+ assert_equal(nil, t.get_token)
109
+
110
+ t = LangScan::D::Tokenizer.new('`\``')
111
+ type, text = t.get_token
112
+ assert_equal('`\`', text)
113
+ assert_equal(:string, type)
114
+ assert_not_equal(nil, t.get_token)
115
+ end
116
+
117
+ def test_d_characters
118
+ t = LangScan::D::Tokenizer.new("'a'")
119
+ type, text = t.get_token
120
+ assert_equal("'a'", text)
121
+ assert_equal(:character, type)
122
+ assert_equal(nil, t.get_token)
123
+
124
+ t = LangScan::D::Tokenizer.new("'\\''")
125
+ type, text = t.get_token
126
+ assert_equal("'\\''", text)
127
+ assert_equal(:character, type)
128
+ assert_equal(nil, t.get_token)
129
+
130
+ t = LangScan::D::Tokenizer.new("'\\x24'")
131
+ type, text = t.get_token
132
+ assert_equal("'\\x24'", text)
133
+ assert_equal(:character, type)
134
+ assert_equal(nil, t.get_token)
135
+
136
+ t = LangScan::D::Tokenizer.new("'\\&quot;'")
137
+ type, text = t.get_token
138
+ assert_equal("'\\&quot;'", text)
139
+ assert_equal(:character, type)
140
+ assert_equal(nil, t.get_token)
141
+ end
142
+
143
+ def test_d_string
144
+ t = LangScan::D::Tokenizer.new(%Q("hello"))
145
+ type, text = t.get_token
146
+ assert_equal(%Q("hello"), text)
147
+ assert_equal(:string, type)
148
+ assert_equal(nil, t.get_token)
149
+
150
+ t = LangScan::D::Tokenizer.new(%Q(`hello`d))
151
+ type, text = t.get_token
152
+ assert_equal(%Q(`hello`d), text)
153
+ assert_equal(:string, type)
154
+ assert_equal(nil, t.get_token)
155
+
156
+ t = LangScan::D::Tokenizer.new(%Q(x"012f"))
157
+ type, text = t.get_token
158
+ assert_equal(%Q(x"012f"), text)
159
+ assert_equal(:string, type)
160
+ assert_equal(nil, t.get_token)
161
+
162
+ t = LangScan::D::Tokenizer.new(%Q(`hello`c))
163
+ type, text = t.get_token
164
+ assert_equal(%Q(`hello`c), text)
165
+ assert_equal(:string, type)
166
+ assert_equal(nil, t.get_token)
167
+
168
+ t = LangScan::D::Tokenizer.new(%Q(r"hello\n"w))
169
+ type, text = t.get_token
170
+ assert_equal(%Q(r"hello\n"w), text)
171
+ assert_equal(:string, type)
172
+ assert_equal(nil, t.get_token)
173
+ end
174
+
175
+ def test_d_comment
176
+ t = LangScan::D::Tokenizer.new("/+ + +/")
177
+ type, text = t.get_token
178
+ assert_equal("/+ + +/", text)
179
+ assert_equal(:comment, type)
180
+ assert_equal(nil, t.get_token)
181
+
182
+ t = LangScan::D::Tokenizer.new("/++/")
183
+ type, text = t.get_token
184
+ assert_equal("/++/", text)
185
+ assert_equal(:comment, type)
186
+ assert_equal(nil, t.get_token)
187
+
188
+ t = LangScan::D::Tokenizer.new("/+ /++/+/")
189
+ type, text = t.get_token
190
+ assert_equal("/+ /++/+/", text)
191
+ assert_equal(:comment, type)
192
+ assert_equal(nil, t.get_token)
193
+
194
+ t = LangScan::D::Tokenizer.new("/+/++//+\n+/+//++/")
195
+ tok = t.get_token
196
+ assert_equal(:comment, tok[0])
197
+ assert_equal("/+/++//+\n+/+/", tok[1])
198
+ assert_equal(1, tok[2])
199
+ assert_equal(0, tok[3])
200
+ assert_equal(0, tok[4])
201
+ assert_equal(2, tok[5])
202
+ assert_equal(4, tok[6])
203
+ assert_equal(13, tok[7])
204
+ end
205
+ end