alex-rkelly 1.0.5 → 1.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/rkelly/constants.rb +1 -1
- data/lib/rkelly/token.rb +4 -0
- data/lib/rkelly/tokenizer.rb +10 -1
- data/test/test_tokenizer.rb +8 -0
- metadata +1 -1
data/lib/rkelly/constants.rb
CHANGED
data/lib/rkelly/token.rb
CHANGED
data/lib/rkelly/tokenizer.rb
CHANGED
@@ -90,7 +90,16 @@ module RKelly
|
|
90
90
|
end
|
91
91
|
end
|
92
92
|
|
93
|
-
|
93
|
+
# To distinguish regular expressions from comments, we require that
|
94
|
+
# regular expressions start with a non * character (ie, not look like
|
95
|
+
# /*foo*/). Note that we can't depend on the length of the match to
|
96
|
+
# correctly distinguish, since `/**/i` is longer if matched as a regular
|
97
|
+
# expression than as matched as a comment.
|
98
|
+
# Incidentally, we're also not matching empty regular expressions
|
99
|
+
# (eg, // and //g). Here we could depend on match length and priority to
|
100
|
+
# determine that these are actually comments, but it turns out to be
|
101
|
+
# easier to not match them in the first place.
|
102
|
+
token(:REGEXP, /\A\/(?:[^\/\r\n\\*]|\\[^\r\n])[^\/\r\n\\]*(?:\\[^\r\n][^\/\r\n\\]*)*\/[gi]*/)
|
94
103
|
token(:S, /\A[\s\r\n]*/m)
|
95
104
|
|
96
105
|
token(:SINGLE_CHAR, /\A./) do |type, value|
|
data/test/test_tokenizer.rb
CHANGED
@@ -127,6 +127,14 @@ class TokenizerTest < Test::Unit::TestCase
|
|
127
127
|
end
|
128
128
|
end
|
129
129
|
|
130
|
+
def test_regular_expression_is_not_found_if_block_comment_with_re_modifier
|
131
|
+
tokens = @tokenizer.tokenize("/**/i")
|
132
|
+
assert_tokens([
|
133
|
+
[:COMMENT, "/**/"],
|
134
|
+
[:IDENT, "i"]
|
135
|
+
], tokens)
|
136
|
+
end
|
137
|
+
|
130
138
|
def test_comment_assign
|
131
139
|
tokens = @tokenizer.tokenize("foo = /**/;")
|
132
140
|
assert_tokens([
|