rkelly-turbo 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (205) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.rdoc +38 -0
  3. data/Manifest.txt +203 -0
  4. data/README.rdoc +135 -0
  5. data/Rakefile +41 -0
  6. data/lib/parser.y +883 -0
  7. data/lib/rkelly.rb +14 -0
  8. data/lib/rkelly/char_pos.rb +39 -0
  9. data/lib/rkelly/char_range.rb +33 -0
  10. data/lib/rkelly/constants.rb +3 -0
  11. data/lib/rkelly/generated_parser.rb +3380 -0
  12. data/lib/rkelly/js.rb +14 -0
  13. data/lib/rkelly/js/array.rb +15 -0
  14. data/lib/rkelly/js/base.rb +91 -0
  15. data/lib/rkelly/js/boolean.rb +21 -0
  16. data/lib/rkelly/js/function.rb +39 -0
  17. data/lib/rkelly/js/function_prototype.rb +15 -0
  18. data/lib/rkelly/js/global_object.rb +52 -0
  19. data/lib/rkelly/js/math.rb +10 -0
  20. data/lib/rkelly/js/nan.rb +18 -0
  21. data/lib/rkelly/js/number.rb +22 -0
  22. data/lib/rkelly/js/object.rb +30 -0
  23. data/lib/rkelly/js/object_prototype.rb +14 -0
  24. data/lib/rkelly/js/property.rb +20 -0
  25. data/lib/rkelly/js/scope.rb +6 -0
  26. data/lib/rkelly/js/string.rb +21 -0
  27. data/lib/rkelly/lexeme.rb +18 -0
  28. data/lib/rkelly/nodes.rb +25 -0
  29. data/lib/rkelly/nodes/binary_node.rb +18 -0
  30. data/lib/rkelly/nodes/bracket_accessor_node.rb +11 -0
  31. data/lib/rkelly/nodes/case_clause_node.rb +11 -0
  32. data/lib/rkelly/nodes/comma_node.rb +11 -0
  33. data/lib/rkelly/nodes/conditional_node.rb +11 -0
  34. data/lib/rkelly/nodes/dot_accessor_node.rb +11 -0
  35. data/lib/rkelly/nodes/for_in_node.rb +12 -0
  36. data/lib/rkelly/nodes/for_node.rb +13 -0
  37. data/lib/rkelly/nodes/function_call_node.rb +16 -0
  38. data/lib/rkelly/nodes/function_decl_node.rb +6 -0
  39. data/lib/rkelly/nodes/function_expr_node.rb +12 -0
  40. data/lib/rkelly/nodes/if_node.rb +12 -0
  41. data/lib/rkelly/nodes/label_node.rb +11 -0
  42. data/lib/rkelly/nodes/new_expr_node.rb +11 -0
  43. data/lib/rkelly/nodes/node.rb +116 -0
  44. data/lib/rkelly/nodes/not_strict_equal_node.rb +6 -0
  45. data/lib/rkelly/nodes/op_equal_node.rb +16 -0
  46. data/lib/rkelly/nodes/postfix_node.rb +11 -0
  47. data/lib/rkelly/nodes/prefix_node.rb +6 -0
  48. data/lib/rkelly/nodes/property_node.rb +13 -0
  49. data/lib/rkelly/nodes/resolve_node.rb +19 -0
  50. data/lib/rkelly/nodes/strict_equal_node.rb +6 -0
  51. data/lib/rkelly/nodes/try_node.rb +13 -0
  52. data/lib/rkelly/nodes/var_decl_node.rb +15 -0
  53. data/lib/rkelly/parser.rb +106 -0
  54. data/lib/rkelly/runtime.rb +36 -0
  55. data/lib/rkelly/runtime/ruby_function.rb +13 -0
  56. data/lib/rkelly/runtime/scope_chain.rb +57 -0
  57. data/lib/rkelly/syntax_error.rb +4 -0
  58. data/lib/rkelly/token.rb +27 -0
  59. data/lib/rkelly/tokenizer.rb +255 -0
  60. data/lib/rkelly/visitable.rb +31 -0
  61. data/lib/rkelly/visitors.rb +4 -0
  62. data/lib/rkelly/visitors/dot_visitor.rb +228 -0
  63. data/lib/rkelly/visitors/ecma_visitor.rb +328 -0
  64. data/lib/rkelly/visitors/enumerable_visitor.rb +18 -0
  65. data/lib/rkelly/visitors/evaluation_visitor.rb +419 -0
  66. data/lib/rkelly/visitors/function_visitor.rb +46 -0
  67. data/lib/rkelly/visitors/pointcut_visitor.rb +31 -0
  68. data/lib/rkelly/visitors/real_sexp_visitor.rb +16 -0
  69. data/lib/rkelly/visitors/sexp_visitor.rb +373 -0
  70. data/lib/rkelly/visitors/visitor.rb +149 -0
  71. data/test/ecma_script_test_case.rb +21 -0
  72. data/test/execute_test_case.rb +16 -0
  73. data/test/execution_contexts/test_10_1_3-1.rb +32 -0
  74. data/test/expressions/test_11_3_1.rb +64 -0
  75. data/test/expressions/test_11_3_2.rb +64 -0
  76. data/test/expressions/test_11_4_2.rb +13 -0
  77. data/test/expressions/test_11_4_3.rb +52 -0
  78. data/test/expressions/test_11_4_4.rb +68 -0
  79. data/test/expressions/test_11_4_5.rb +69 -0
  80. data/test/expressions/test_11_4_6.rb +94 -0
  81. data/test/expressions/test_11_4_8.rb +28 -0
  82. data/test/expressions/test_11_4_9.rb +103 -0
  83. data/test/expressions/test_11_5_1.rb +51 -0
  84. data/test/expressions/test_11_5_2.rb +80 -0
  85. data/test/expressions/test_11_5_3.rb +88 -0
  86. data/test/expressions/test_11_6_1-1.rb +19 -0
  87. data/test/expressions/test_11_9_1.rb +19 -0
  88. data/test/function/test_15_3_1_1-1.rb +34 -0
  89. data/test/global_object/test_15_1_1_1.rb +29 -0
  90. data/test/global_object/test_15_1_1_2.rb +17 -0
  91. data/test/global_object/test_15_1_1_3.rb +9 -0
  92. data/test/helper.rb +5 -0
  93. data/test/node_test_case.rb +11 -0
  94. data/test/object/test_15_2_1_1.rb +257 -0
  95. data/test/object/test_15_2_1_2.rb +21 -0
  96. data/test/object/test_15_2_2_1.rb +52 -0
  97. data/test/statements/test_12_5-1.rb +27 -0
  98. data/test/test_add_node.rb +8 -0
  99. data/test/test_arguments_node.rb +8 -0
  100. data/test/test_array_node.rb +9 -0
  101. data/test/test_assign_expr_node.rb +8 -0
  102. data/test/test_automatic_semicolon_insertion.rb +137 -0
  103. data/test/test_bit_and_node.rb +8 -0
  104. data/test/test_bit_or_node.rb +8 -0
  105. data/test/test_bit_x_or_node.rb +8 -0
  106. data/test/test_bitwise_not_node.rb +8 -0
  107. data/test/test_block_node.rb +14 -0
  108. data/test/test_bracket_accessor_node.rb +16 -0
  109. data/test/test_break_node.rb +11 -0
  110. data/test/test_case_block_node.rb +11 -0
  111. data/test/test_case_clause_node.rb +15 -0
  112. data/test/test_char_pos.rb +39 -0
  113. data/test/test_char_range.rb +29 -0
  114. data/test/test_comma_node.rb +13 -0
  115. data/test/test_comments.rb +45 -0
  116. data/test/test_conditional_node.rb +17 -0
  117. data/test/test_const_statement_node.rb +14 -0
  118. data/test/test_continue_node.rb +11 -0
  119. data/test/test_delete_node.rb +8 -0
  120. data/test/test_divide_node.rb +8 -0
  121. data/test/test_do_while_node.rb +13 -0
  122. data/test/test_dot_accessor_node.rb +9 -0
  123. data/test/test_ecma_visitor.rb +213 -0
  124. data/test/test_element_node.rb +8 -0
  125. data/test/test_empty_statement_node.rb +8 -0
  126. data/test/test_equal_node.rb +8 -0
  127. data/test/test_evaluation_visitor.rb +66 -0
  128. data/test/test_expression_statement_node.rb +10 -0
  129. data/test/test_false_node.rb +8 -0
  130. data/test/test_for_in_node.rb +17 -0
  131. data/test/test_for_node.rb +24 -0
  132. data/test/test_function_body_node.rb +8 -0
  133. data/test/test_function_call_node.rb +10 -0
  134. data/test/test_function_decl_node.rb +16 -0
  135. data/test/test_function_expr_node.rb +16 -0
  136. data/test/test_function_visitor.rb +26 -0
  137. data/test/test_getter_property_node.rb +10 -0
  138. data/test/test_global_object.rb +49 -0
  139. data/test/test_greater_node.rb +8 -0
  140. data/test/test_greater_or_equal_node.rb +8 -0
  141. data/test/test_if_node.rb +17 -0
  142. data/test/test_in_node.rb +8 -0
  143. data/test/test_instance_of_node.rb +8 -0
  144. data/test/test_label_node.rb +13 -0
  145. data/test/test_left_shift_node.rb +8 -0
  146. data/test/test_less_node.rb +8 -0
  147. data/test/test_less_or_equal_node.rb +8 -0
  148. data/test/test_line_number.rb +81 -0
  149. data/test/test_logical_and_node.rb +8 -0
  150. data/test/test_logical_not_node.rb +8 -0
  151. data/test/test_logical_or_node.rb +8 -0
  152. data/test/test_modulus_node.rb +8 -0
  153. data/test/test_multiply_node.rb +8 -0
  154. data/test/test_new_expr_node.rb +9 -0
  155. data/test/test_not_equal_node.rb +8 -0
  156. data/test/test_not_strict_equal_node.rb +8 -0
  157. data/test/test_null_node.rb +8 -0
  158. data/test/test_number_node.rb +8 -0
  159. data/test/test_object_literal_node.rb +9 -0
  160. data/test/test_op_and_equal_node.rb +10 -0
  161. data/test/test_op_divide_equal_node.rb +10 -0
  162. data/test/test_op_equal_node.rb +10 -0
  163. data/test/test_op_l_shift_equal_node.rb +10 -0
  164. data/test/test_op_minus_equal_node.rb +10 -0
  165. data/test/test_op_mod_equal_node.rb +10 -0
  166. data/test/test_op_multiply_equal_node.rb +10 -0
  167. data/test/test_op_or_equal_node.rb +10 -0
  168. data/test/test_op_plus_equal_node.rb +10 -0
  169. data/test/test_op_r_shift_equal_node.rb +10 -0
  170. data/test/test_op_u_r_shift_equal_node.rb +10 -0
  171. data/test/test_op_x_or_equal_node.rb +10 -0
  172. data/test/test_parameter_node.rb +8 -0
  173. data/test/test_parser.rb +1434 -0
  174. data/test/test_pointcut_visitor.rb +34 -0
  175. data/test/test_postfix_node.rb +8 -0
  176. data/test/test_prefix_node.rb +8 -0
  177. data/test/test_property_node.rb +8 -0
  178. data/test/test_regexp_node.rb +8 -0
  179. data/test/test_resolve_node.rb +22 -0
  180. data/test/test_return_node.rb +11 -0
  181. data/test/test_right_shift_node.rb +8 -0
  182. data/test/test_rkelly.rb +19 -0
  183. data/test/test_runtime.rb +12 -0
  184. data/test/test_scope_chain.rb +50 -0
  185. data/test/test_setter_property_node.rb +10 -0
  186. data/test/test_source_elements.rb +9 -0
  187. data/test/test_strict_equal_node.rb +8 -0
  188. data/test/test_string_node.rb +8 -0
  189. data/test/test_subtract_node.rb +8 -0
  190. data/test/test_switch_node.rb +12 -0
  191. data/test/test_this_node.rb +8 -0
  192. data/test/test_throw_node.rb +7 -0
  193. data/test/test_tokenizer.rb +285 -0
  194. data/test/test_true_node.rb +8 -0
  195. data/test/test_try_node.rb +59 -0
  196. data/test/test_type_of_node.rb +8 -0
  197. data/test/test_unary_minus_node.rb +8 -0
  198. data/test/test_unary_plus_node.rb +8 -0
  199. data/test/test_unsigned_right_shift_node.rb +8 -0
  200. data/test/test_var_decl_node.rb +21 -0
  201. data/test/test_var_statement_node.rb +14 -0
  202. data/test/test_void_node.rb +8 -0
  203. data/test/test_while_node.rb +15 -0
  204. data/test/test_with_node.rb +8 -0
  205. metadata +293 -0
@@ -0,0 +1,34 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class PointcutVisitorTest < Test::Unit::TestCase
4
+ include RKelly::Visitors
5
+ include RKelly::Nodes
6
+
7
+ def setup
8
+ @parser = RKelly::Parser.new
9
+ end
10
+
11
+ def test_visit_NumberNode
12
+ ast = @parser.parse('Element.update(10, 10)')
13
+ assert_equal(2, ast.pointcut('10').matches.length)
14
+ end
15
+
16
+ def test_visit_RegexpNode
17
+ ast = @parser.parse('Element.update(/asdf/, /asdf/)')
18
+ assert_equal(2, ast.pointcut('/asdf/').matches.length)
19
+ end
20
+
21
+ def test_visit_ContinueNode
22
+ ast = @parser.parse('function foo() { continue; }')
23
+ cut = ast.pointcut('continue')
24
+ assert_equal(1, cut.matches.length)
25
+ assert cut.matches.first.is_a?(ContinueNode)
26
+ end
27
+
28
+ def test_try_catch
29
+ ast = @parser.parse('try { Element.update(10, 10); } catch(e) { }')
30
+ assert_equal(1, ast.pointcut('Element.update(10, 10)').matches.length)
31
+ ast = @parser.parse('try { Element.update("foo", "bar"); } catch(e) { }')
32
+ assert_equal(1, ast.pointcut('Element.update(String, String)').matches.length)
33
+ end
34
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class PostfixNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = PostfixNode.new(NumberNode.new(10), '++')
6
+ assert_sexp([:postfix, [:lit, 10], '++'], node)
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class PrefixNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = PrefixNode.new(NumberNode.new(10), '++')
6
+ assert_sexp([:prefix, [:lit, 10], '++'], node)
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class PropertyNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = PropertyNode.new('foo', NumberNode.new(10))
6
+ assert_sexp([:property, :foo, [:lit, 10]], node)
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class RegexpNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = RegexpNode.new('/yay!/')
6
+ assert_sexp [:lit, '/yay!/'], node
7
+ end
8
+ end
@@ -0,0 +1,22 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class ResolveNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = ResolveNode.new('foo')
6
+ assert_sexp [:resolve, 'foo'], node
7
+ end
8
+
9
+ def test_match
10
+ node = ResolveNode.new('foo')
11
+ node2 = ResolveNode.new('foo')
12
+ assert(node =~ node2)
13
+
14
+ assert(node !~ NumberNode.new(10))
15
+ end
16
+
17
+ def test_is_a
18
+ node = ResolveNode.new('foo')
19
+ node3 = ResolveNode.new('String')
20
+ assert(node3 =~ node)
21
+ end
22
+ end
@@ -0,0 +1,11 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class ReturnNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = ReturnNode.new(nil)
6
+ assert_sexp([:return], node)
7
+
8
+ node = ReturnNode.new(NumberNode.new(10))
9
+ assert_sexp([:return, [:lit, 10]], node)
10
+ end
11
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class RightShiftNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = RightShiftNode.new(NumberNode.new(5), NumberNode.new(10))
6
+ assert_sexp([:rshift, [:lit, 5], [:lit, 10]], node)
7
+ end
8
+ end
@@ -0,0 +1,19 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class RKellyTest < Test::Unit::TestCase
4
+ def test_array_access
5
+ assert_sexp(
6
+ [
7
+ [:var,
8
+ [[:var_decl, :a,
9
+ [:assign, [:bracket_access, [:resolve, "foo"], [:lit, 10]]],
10
+ ]]
11
+ ]
12
+ ],
13
+ RKelly.parse('var a = foo[10];'))
14
+ end
15
+
16
+ def assert_sexp(expected, node)
17
+ assert_equal(expected, node.to_sexp)
18
+ end
19
+ end
@@ -0,0 +1,12 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class RuntimeTest < Test::Unit::TestCase
4
+ def setup
5
+ @runtime = RKelly::Runtime.new
6
+ end
7
+
8
+ def test_call_function
9
+ @runtime.execute("function foo(a) { return a + 2; }")
10
+ assert_equal(12, @runtime.call_function("foo", 10))
11
+ end
12
+ end
@@ -0,0 +1,50 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class TestScopeChain < Test::Unit::TestCase
4
+ def setup
5
+ @scope_chain = RKelly::Runtime::ScopeChain.new
6
+ scope_1 = RKelly::JS::Scope.new
7
+ scope_1.properties[:foo] = 1
8
+ scope_2 = RKelly::JS::Scope.new
9
+ scope_2.properties[:bar] = 10
10
+ @scope_chain << scope_1
11
+ @scope_chain << scope_2
12
+ end
13
+
14
+ def test_global_object_in_chain
15
+ assert @scope_chain.has_property?('NaN')
16
+ end
17
+
18
+ def test_has_property
19
+ assert @scope_chain.has_property?(:foo)
20
+ assert @scope_chain.has_property?(:bar)
21
+ assert !@scope_chain.has_property?(:baz)
22
+ @scope_chain.pop
23
+ assert @scope_chain.has_property?(:bar).nil?
24
+ end
25
+
26
+ def test_find_property
27
+ assert_equal(10, @scope_chain[:bar])
28
+ assert_equal(1, @scope_chain[:foo])
29
+ @scope_chain.pop
30
+ assert(!@scope_chain.has_property?(:bar))
31
+ end
32
+
33
+ def test_add_property
34
+ assert !@scope_chain.has_property?(:baz)
35
+ @scope_chain[:baz] = 10
36
+ assert @scope_chain.has_property?(:baz)
37
+ @scope_chain.pop
38
+ assert !@scope_chain.has_property?(:baz)
39
+ end
40
+
41
+ def test_chain_in_block
42
+ assert !@scope_chain.has_property?(:baz)
43
+ @scope_chain.new_scope { |chain|
44
+ chain[:baz] = 10
45
+ assert @scope_chain.has_property?(:baz)
46
+ assert chain.has_property?(:baz)
47
+ }
48
+ assert @scope_chain.has_property?(:baz).nil?
49
+ end
50
+ end
@@ -0,0 +1,10 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class SetterPropertyNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ body = FunctionBodyNode.new(SourceElementsNode.new([]))
6
+ function = FunctionExprNode.new(nil, body)
7
+ node = SetterPropertyNode.new('foo', function)
8
+ assert_sexp([:setter, :foo, [:func_expr, nil, [], [:func_body, []]]], node)
9
+ end
10
+ end
@@ -0,0 +1,9 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class SourceElementListTest < NodeTestCase
4
+ def test_to_sexp
5
+ num = NumberNode.new(10)
6
+ node = SourceElementsNode.new([num, num])
7
+ assert_sexp [[:lit, 10], [:lit, 10]], node
8
+ end
9
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class StrictEqualNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = StrictEqualNode.new(NumberNode.new(5), NumberNode.new(10))
6
+ assert_sexp([:strict_equal, [:lit, 5], [:lit, 10]], node)
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class StringNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = StringNode.new('"asdf"')
6
+ assert_sexp [:str, '"asdf"'], node
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class SubtractNodeTest < NodeTestCase
4
+ def test_subtract
5
+ node = SubtractNode.new(NumberNode.new(5), NumberNode.new(10))
6
+ assert_sexp([:subtract, [:lit, 5], [:lit, 10]], node)
7
+ end
8
+ end
@@ -0,0 +1,12 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class SwitchNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ clause = CaseClauseNode.new( ResolveNode.new('foo'),
6
+ SourceElementsNode.new([ResolveNode.new('bar')]))
7
+ block = CaseBlockNode.new([clause])
8
+ node = SwitchNode.new(ResolveNode.new('o'), block)
9
+ assert_sexp([:switch, [:resolve, 'o'],[:case_block, [[:case, [:resolve, 'foo'], [[:resolve, 'bar']]]]]],
10
+ node)
11
+ end
12
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class ThisNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = ThisNode.new('this')
6
+ assert_sexp([:this], node)
7
+ end
8
+ end
@@ -0,0 +1,7 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class ThrowNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ assert_sexp([:throw, [:lit, 10]], ThrowNode.new(NumberNode.new(10)))
6
+ end
7
+ end
@@ -0,0 +1,285 @@
1
+ # -*- coding: utf-8 -*-
2
+ require File.dirname(__FILE__) + "/helper"
3
+
4
+ class TokenizerTest < Test::Unit::TestCase
5
+ def setup
6
+ @tokenizer = RKelly::Tokenizer.new
7
+ end
8
+
9
+ {
10
+ :space => " ",
11
+ :tab => "\t",
12
+ :form_feed => "\f",
13
+ :vertical_tab => "\v",
14
+ :no_break_space => [0x00A0].pack("U"),
15
+ :ogham_space_mark => [0x1680].pack("U"),
16
+ :en_quad => [0x2000].pack("U"),
17
+ :em_quad => [0x2001].pack("U"),
18
+ :en_space => [0x2002].pack("U"),
19
+ :em_space => [0x2003].pack("U"),
20
+ :three_per_em_space => [0x2004].pack("U"),
21
+ :four_per_em_space => [0x2005].pack("U"),
22
+ :six_per_em_space => [0x2006].pack("U"),
23
+ :figure_space => [0x2007].pack("U"),
24
+ :punctuation_space => [0x2008].pack("U"),
25
+ :thin_space => [0x2009].pack("U"),
26
+ :hair_space => [0x200a].pack("U"),
27
+ :narrow_no_break_space => [0x202f].pack("U"),
28
+ :medium_mathematical_space => [0x205f].pack("U"),
29
+ :ideographic_space => [0x3000].pack("U"),
30
+
31
+ # Line terminators
32
+ :newline => "\n",
33
+ :carriage_return => "\r",
34
+ :line_separator => [0x2028].pack("U"),
35
+ :paragraph_separator => [0x2029].pack("U"),
36
+ }.each do |name, char|
37
+ define_method(:"test_whitespace_#{name}") do
38
+ assert_equal([[:S, char]], @tokenizer.tokenize(char))
39
+ end
40
+ end
41
+
42
+ def assert_tokens(expected, actual)
43
+ assert_equal(expected, actual.select { |x| x[0] != :S })
44
+ end
45
+
46
+ def test_comments
47
+ tokens = @tokenizer.tokenize("/** Fooo */")
48
+ assert_tokens([[:COMMENT, '/** Fooo */']], tokens)
49
+ end
50
+
51
+ def test_string_single_quote
52
+ tokens = @tokenizer.tokenize("foo = 'hello world';")
53
+ assert_tokens([
54
+ [:IDENT, 'foo'],
55
+ ['=', '='],
56
+ [:STRING, "'hello world'"],
57
+ [';', ';'],
58
+ ], tokens)
59
+ end
60
+
61
+ def test_string_double_quote
62
+ tokens = @tokenizer.tokenize('foo = "hello world";')
63
+ assert_tokens([
64
+ [:IDENT, 'foo'],
65
+ ['=', '='],
66
+ [:STRING, '"hello world"'],
67
+ [';', ';'],
68
+ ], tokens)
69
+ end
70
+
71
+ def test_number_parse
72
+ tokens = @tokenizer.tokenize('3.')
73
+ assert_tokens([[:NUMBER, 3.0]], tokens)
74
+
75
+ tokens = @tokenizer.tokenize('3.e1')
76
+ assert_tokens([[:NUMBER, 30]], tokens)
77
+
78
+ tokens = @tokenizer.tokenize('.001')
79
+ assert_tokens([[:NUMBER, 0.001]], tokens)
80
+
81
+ tokens = @tokenizer.tokenize('3.e-1')
82
+ assert_tokens([[:NUMBER, 0.30]], tokens)
83
+ end
84
+
85
+ def test_identifier
86
+ tokens = @tokenizer.tokenize("foo")
87
+ assert_tokens([[:IDENT, 'foo']], tokens)
88
+ end
89
+
90
+ def test_ignore_identifier
91
+ tokens = @tokenizer.tokenize("0foo")
92
+ assert_tokens([[:NUMBER, 0], [:IDENT, 'foo']], tokens)
93
+ end
94
+
95
+ def test_increment
96
+ tokens = @tokenizer.tokenize("foo += 1;")
97
+ assert_tokens([
98
+ [:IDENT, 'foo'],
99
+ [:PLUSEQUAL, '+='],
100
+ [:NUMBER, 1],
101
+ [';', ';'],
102
+ ], tokens)
103
+ end
104
+
105
+ def test_regular_expression
106
+ tokens = @tokenizer.tokenize("foo = /=asdf/;")
107
+ assert_tokens([
108
+ [:IDENT, 'foo'],
109
+ ['=', '='],
110
+ [:REGEXP, '/=asdf/'],
111
+ [';', ';'],
112
+ ], tokens)
113
+ end
114
+
115
+ def test_regular_expression_invalid
116
+ tokens = @tokenizer.tokenize("foo = (1 / 2) / 3")
117
+ assert_tokens([[:IDENT, "foo"],
118
+ ["=", "="],
119
+ ["(", "("],
120
+ [:NUMBER, 1],
121
+ ["/", "/"],
122
+ [:NUMBER, 2],
123
+ [")", ")"],
124
+ ["/", "/"],
125
+ [:NUMBER, 3]
126
+ ], tokens)
127
+ end
128
+
129
+ def test_regular_expression_escape
130
+ tokens = @tokenizer.tokenize('foo = /\/asdf/gi;')
131
+ assert_tokens([
132
+ [:IDENT, 'foo'],
133
+ ['=', '='],
134
+ [:REGEXP, '/\/asdf/gi'],
135
+ [';', ';'],
136
+ ], tokens)
137
+ end
138
+
139
+ def test_regular_expression_with_slash_inside_charset
140
+ tokens = @tokenizer.tokenize('foo = /[/]/;')
141
+ assert_tokens([
142
+ [:IDENT, 'foo'],
143
+ ['=', '='],
144
+ [:REGEXP, '/[/]/'],
145
+ [';', ';'],
146
+ ], tokens)
147
+ end
148
+
149
+ def test_regular_expression_is_not_found_if_prev_token_implies_division
150
+ {:IDENT => 'foo',
151
+ :TRUE => 'true',
152
+ :NUMBER => 1,
153
+ ')' => ')',
154
+ ']' => ']',
155
+ '}' => '}'}.each do |name, value|
156
+ tokens = @tokenizer.tokenize("#{value}/2/3")
157
+ assert_tokens([
158
+ [name, value],
159
+ ["/", "/"],
160
+ [:NUMBER, 2],
161
+ ["/", "/"],
162
+ [:NUMBER, 3],
163
+ ], tokens)
164
+ end
165
+ end
166
+
167
+ def test_regular_expression_is_found_if_prev_token_is_non_literal_keyword
168
+ {:RETURN => 'return',
169
+ :THROW => 'throw'}.each do |name, value|
170
+ tokens = @tokenizer.tokenize("#{value}/2/")
171
+ assert_tokens([
172
+ [name, value],
173
+ [:REGEXP, "/2/"],
174
+ ], tokens)
175
+ end
176
+ end
177
+
178
+ def test_regular_expression_is_not_found_if_block_comment_with_re_modifier
179
+ tokens = @tokenizer.tokenize("/**/i")
180
+ assert_tokens([
181
+ [:COMMENT, "/**/"],
182
+ [:IDENT, "i"]
183
+ ], tokens)
184
+ end
185
+
186
+ def test_comment_assign
187
+ tokens = @tokenizer.tokenize("foo = /**/;")
188
+ assert_tokens([
189
+ [:IDENT, 'foo'],
190
+ ['=', '='],
191
+ [:COMMENT, '/**/'],
192
+ [';', ';'],
193
+ ], tokens)
194
+
195
+ tokens = @tokenizer.tokenize("foo = //;")
196
+ assert_tokens([
197
+ [:IDENT, 'foo'],
198
+ ['=', '='],
199
+ [:COMMENT, '//;'],
200
+ ], tokens)
201
+ end
202
+
203
+ def test_unicode_string
204
+ tokens = @tokenizer.tokenize("foo = 'öäüõ';")
205
+ assert_tokens([
206
+ [:IDENT, 'foo'],
207
+ ['=', '='],
208
+ [:STRING, "'öäüõ'"],
209
+ [';', ';'],
210
+ ], tokens)
211
+ end
212
+
213
+ def test_unicode_regex
214
+ tokens = @tokenizer.tokenize("foo = /öäüõ/;")
215
+ assert_tokens([
216
+ [:IDENT, 'foo'],
217
+ ['=', '='],
218
+ [:REGEXP, "/öäüõ/"],
219
+ [';', ';'],
220
+ ], tokens)
221
+ end
222
+
223
+ %w{
224
+ break case catch continue default delete do else finally for function
225
+ if in instanceof new return switch this throw try typeof var void while
226
+ with
227
+
228
+ const true false null debugger
229
+ }.each do |kw|
230
+ define_method(:"test_keyword_#{kw}") do
231
+ tokens = @tokenizer.tokenize(kw)
232
+ assert_equal 1, tokens.length
233
+ assert_equal([[kw.upcase.to_sym, kw]], tokens)
234
+ end
235
+ end
236
+
237
+ %w{
238
+ class enum extends super export import
239
+ }.each do |rw|
240
+ define_method(:"test_future_reserved_word_#{rw}_is_reserved") do
241
+ tokens = @tokenizer.tokenize(rw)
242
+ assert_equal 1, tokens.length
243
+ assert_equal([[:RESERVED, rw]], tokens)
244
+ end
245
+ end
246
+
247
+ %w{
248
+ implements let private public yield
249
+ interface package protected static
250
+ }.each do |rw|
251
+ define_method(:"test_future_reserved_word_#{rw}_is_identifier") do
252
+ tokens = @tokenizer.tokenize(rw)
253
+ assert_equal 1, tokens.length
254
+ assert_equal([[:IDENT, rw]], tokens)
255
+ end
256
+ end
257
+
258
+ {
259
+ '==' => :EQEQ,
260
+ '!=' => :NE,
261
+ '===' => :STREQ,
262
+ '!==' => :STRNEQ,
263
+ '<=' => :LE,
264
+ '>=' => :GE,
265
+ '||' => :OR,
266
+ '&&' => :AND,
267
+ '++' => :PLUSPLUS,
268
+ '--' => :MINUSMINUS,
269
+ '<<' => :LSHIFT,
270
+ '>>' => :RSHIFT,
271
+ '>>>' => :URSHIFT,
272
+ '+=' => :PLUSEQUAL,
273
+ '-=' => :MINUSEQUAL,
274
+ '*=' => :MULTEQUAL,
275
+ 'null' => :NULL,
276
+ 'true' => :TRUE,
277
+ 'false' => :FALSE,
278
+ }.each do |punctuator, sym|
279
+ define_method(:"test_punctuator_#{sym}") do
280
+ tokens = @tokenizer.tokenize(punctuator)
281
+ assert_equal 1, tokens.length
282
+ assert_equal([[sym, punctuator]], tokens)
283
+ end
284
+ end
285
+ end