rkelly-turbo 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.rdoc +38 -0
  3. data/Manifest.txt +203 -0
  4. data/README.rdoc +135 -0
  5. data/Rakefile +41 -0
  6. data/lib/parser.y +883 -0
  7. data/lib/rkelly.rb +14 -0
  8. data/lib/rkelly/char_pos.rb +39 -0
  9. data/lib/rkelly/char_range.rb +33 -0
  10. data/lib/rkelly/constants.rb +3 -0
  11. data/lib/rkelly/generated_parser.rb +3380 -0
  12. data/lib/rkelly/js.rb +14 -0
  13. data/lib/rkelly/js/array.rb +15 -0
  14. data/lib/rkelly/js/base.rb +91 -0
  15. data/lib/rkelly/js/boolean.rb +21 -0
  16. data/lib/rkelly/js/function.rb +39 -0
  17. data/lib/rkelly/js/function_prototype.rb +15 -0
  18. data/lib/rkelly/js/global_object.rb +52 -0
  19. data/lib/rkelly/js/math.rb +10 -0
  20. data/lib/rkelly/js/nan.rb +18 -0
  21. data/lib/rkelly/js/number.rb +22 -0
  22. data/lib/rkelly/js/object.rb +30 -0
  23. data/lib/rkelly/js/object_prototype.rb +14 -0
  24. data/lib/rkelly/js/property.rb +20 -0
  25. data/lib/rkelly/js/scope.rb +6 -0
  26. data/lib/rkelly/js/string.rb +21 -0
  27. data/lib/rkelly/lexeme.rb +18 -0
  28. data/lib/rkelly/nodes.rb +25 -0
  29. data/lib/rkelly/nodes/binary_node.rb +18 -0
  30. data/lib/rkelly/nodes/bracket_accessor_node.rb +11 -0
  31. data/lib/rkelly/nodes/case_clause_node.rb +11 -0
  32. data/lib/rkelly/nodes/comma_node.rb +11 -0
  33. data/lib/rkelly/nodes/conditional_node.rb +11 -0
  34. data/lib/rkelly/nodes/dot_accessor_node.rb +11 -0
  35. data/lib/rkelly/nodes/for_in_node.rb +12 -0
  36. data/lib/rkelly/nodes/for_node.rb +13 -0
  37. data/lib/rkelly/nodes/function_call_node.rb +16 -0
  38. data/lib/rkelly/nodes/function_decl_node.rb +6 -0
  39. data/lib/rkelly/nodes/function_expr_node.rb +12 -0
  40. data/lib/rkelly/nodes/if_node.rb +12 -0
  41. data/lib/rkelly/nodes/label_node.rb +11 -0
  42. data/lib/rkelly/nodes/new_expr_node.rb +11 -0
  43. data/lib/rkelly/nodes/node.rb +116 -0
  44. data/lib/rkelly/nodes/not_strict_equal_node.rb +6 -0
  45. data/lib/rkelly/nodes/op_equal_node.rb +16 -0
  46. data/lib/rkelly/nodes/postfix_node.rb +11 -0
  47. data/lib/rkelly/nodes/prefix_node.rb +6 -0
  48. data/lib/rkelly/nodes/property_node.rb +13 -0
  49. data/lib/rkelly/nodes/resolve_node.rb +19 -0
  50. data/lib/rkelly/nodes/strict_equal_node.rb +6 -0
  51. data/lib/rkelly/nodes/try_node.rb +13 -0
  52. data/lib/rkelly/nodes/var_decl_node.rb +15 -0
  53. data/lib/rkelly/parser.rb +106 -0
  54. data/lib/rkelly/runtime.rb +36 -0
  55. data/lib/rkelly/runtime/ruby_function.rb +13 -0
  56. data/lib/rkelly/runtime/scope_chain.rb +57 -0
  57. data/lib/rkelly/syntax_error.rb +4 -0
  58. data/lib/rkelly/token.rb +27 -0
  59. data/lib/rkelly/tokenizer.rb +255 -0
  60. data/lib/rkelly/visitable.rb +31 -0
  61. data/lib/rkelly/visitors.rb +4 -0
  62. data/lib/rkelly/visitors/dot_visitor.rb +228 -0
  63. data/lib/rkelly/visitors/ecma_visitor.rb +328 -0
  64. data/lib/rkelly/visitors/enumerable_visitor.rb +18 -0
  65. data/lib/rkelly/visitors/evaluation_visitor.rb +419 -0
  66. data/lib/rkelly/visitors/function_visitor.rb +46 -0
  67. data/lib/rkelly/visitors/pointcut_visitor.rb +31 -0
  68. data/lib/rkelly/visitors/real_sexp_visitor.rb +16 -0
  69. data/lib/rkelly/visitors/sexp_visitor.rb +373 -0
  70. data/lib/rkelly/visitors/visitor.rb +149 -0
  71. data/test/ecma_script_test_case.rb +21 -0
  72. data/test/execute_test_case.rb +16 -0
  73. data/test/execution_contexts/test_10_1_3-1.rb +32 -0
  74. data/test/expressions/test_11_3_1.rb +64 -0
  75. data/test/expressions/test_11_3_2.rb +64 -0
  76. data/test/expressions/test_11_4_2.rb +13 -0
  77. data/test/expressions/test_11_4_3.rb +52 -0
  78. data/test/expressions/test_11_4_4.rb +68 -0
  79. data/test/expressions/test_11_4_5.rb +69 -0
  80. data/test/expressions/test_11_4_6.rb +94 -0
  81. data/test/expressions/test_11_4_8.rb +28 -0
  82. data/test/expressions/test_11_4_9.rb +103 -0
  83. data/test/expressions/test_11_5_1.rb +51 -0
  84. data/test/expressions/test_11_5_2.rb +80 -0
  85. data/test/expressions/test_11_5_3.rb +88 -0
  86. data/test/expressions/test_11_6_1-1.rb +19 -0
  87. data/test/expressions/test_11_9_1.rb +19 -0
  88. data/test/function/test_15_3_1_1-1.rb +34 -0
  89. data/test/global_object/test_15_1_1_1.rb +29 -0
  90. data/test/global_object/test_15_1_1_2.rb +17 -0
  91. data/test/global_object/test_15_1_1_3.rb +9 -0
  92. data/test/helper.rb +5 -0
  93. data/test/node_test_case.rb +11 -0
  94. data/test/object/test_15_2_1_1.rb +257 -0
  95. data/test/object/test_15_2_1_2.rb +21 -0
  96. data/test/object/test_15_2_2_1.rb +52 -0
  97. data/test/statements/test_12_5-1.rb +27 -0
  98. data/test/test_add_node.rb +8 -0
  99. data/test/test_arguments_node.rb +8 -0
  100. data/test/test_array_node.rb +9 -0
  101. data/test/test_assign_expr_node.rb +8 -0
  102. data/test/test_automatic_semicolon_insertion.rb +137 -0
  103. data/test/test_bit_and_node.rb +8 -0
  104. data/test/test_bit_or_node.rb +8 -0
  105. data/test/test_bit_x_or_node.rb +8 -0
  106. data/test/test_bitwise_not_node.rb +8 -0
  107. data/test/test_block_node.rb +14 -0
  108. data/test/test_bracket_accessor_node.rb +16 -0
  109. data/test/test_break_node.rb +11 -0
  110. data/test/test_case_block_node.rb +11 -0
  111. data/test/test_case_clause_node.rb +15 -0
  112. data/test/test_char_pos.rb +39 -0
  113. data/test/test_char_range.rb +29 -0
  114. data/test/test_comma_node.rb +13 -0
  115. data/test/test_comments.rb +45 -0
  116. data/test/test_conditional_node.rb +17 -0
  117. data/test/test_const_statement_node.rb +14 -0
  118. data/test/test_continue_node.rb +11 -0
  119. data/test/test_delete_node.rb +8 -0
  120. data/test/test_divide_node.rb +8 -0
  121. data/test/test_do_while_node.rb +13 -0
  122. data/test/test_dot_accessor_node.rb +9 -0
  123. data/test/test_ecma_visitor.rb +213 -0
  124. data/test/test_element_node.rb +8 -0
  125. data/test/test_empty_statement_node.rb +8 -0
  126. data/test/test_equal_node.rb +8 -0
  127. data/test/test_evaluation_visitor.rb +66 -0
  128. data/test/test_expression_statement_node.rb +10 -0
  129. data/test/test_false_node.rb +8 -0
  130. data/test/test_for_in_node.rb +17 -0
  131. data/test/test_for_node.rb +24 -0
  132. data/test/test_function_body_node.rb +8 -0
  133. data/test/test_function_call_node.rb +10 -0
  134. data/test/test_function_decl_node.rb +16 -0
  135. data/test/test_function_expr_node.rb +16 -0
  136. data/test/test_function_visitor.rb +26 -0
  137. data/test/test_getter_property_node.rb +10 -0
  138. data/test/test_global_object.rb +49 -0
  139. data/test/test_greater_node.rb +8 -0
  140. data/test/test_greater_or_equal_node.rb +8 -0
  141. data/test/test_if_node.rb +17 -0
  142. data/test/test_in_node.rb +8 -0
  143. data/test/test_instance_of_node.rb +8 -0
  144. data/test/test_label_node.rb +13 -0
  145. data/test/test_left_shift_node.rb +8 -0
  146. data/test/test_less_node.rb +8 -0
  147. data/test/test_less_or_equal_node.rb +8 -0
  148. data/test/test_line_number.rb +81 -0
  149. data/test/test_logical_and_node.rb +8 -0
  150. data/test/test_logical_not_node.rb +8 -0
  151. data/test/test_logical_or_node.rb +8 -0
  152. data/test/test_modulus_node.rb +8 -0
  153. data/test/test_multiply_node.rb +8 -0
  154. data/test/test_new_expr_node.rb +9 -0
  155. data/test/test_not_equal_node.rb +8 -0
  156. data/test/test_not_strict_equal_node.rb +8 -0
  157. data/test/test_null_node.rb +8 -0
  158. data/test/test_number_node.rb +8 -0
  159. data/test/test_object_literal_node.rb +9 -0
  160. data/test/test_op_and_equal_node.rb +10 -0
  161. data/test/test_op_divide_equal_node.rb +10 -0
  162. data/test/test_op_equal_node.rb +10 -0
  163. data/test/test_op_l_shift_equal_node.rb +10 -0
  164. data/test/test_op_minus_equal_node.rb +10 -0
  165. data/test/test_op_mod_equal_node.rb +10 -0
  166. data/test/test_op_multiply_equal_node.rb +10 -0
  167. data/test/test_op_or_equal_node.rb +10 -0
  168. data/test/test_op_plus_equal_node.rb +10 -0
  169. data/test/test_op_r_shift_equal_node.rb +10 -0
  170. data/test/test_op_u_r_shift_equal_node.rb +10 -0
  171. data/test/test_op_x_or_equal_node.rb +10 -0
  172. data/test/test_parameter_node.rb +8 -0
  173. data/test/test_parser.rb +1434 -0
  174. data/test/test_pointcut_visitor.rb +34 -0
  175. data/test/test_postfix_node.rb +8 -0
  176. data/test/test_prefix_node.rb +8 -0
  177. data/test/test_property_node.rb +8 -0
  178. data/test/test_regexp_node.rb +8 -0
  179. data/test/test_resolve_node.rb +22 -0
  180. data/test/test_return_node.rb +11 -0
  181. data/test/test_right_shift_node.rb +8 -0
  182. data/test/test_rkelly.rb +19 -0
  183. data/test/test_runtime.rb +12 -0
  184. data/test/test_scope_chain.rb +50 -0
  185. data/test/test_setter_property_node.rb +10 -0
  186. data/test/test_source_elements.rb +9 -0
  187. data/test/test_strict_equal_node.rb +8 -0
  188. data/test/test_string_node.rb +8 -0
  189. data/test/test_subtract_node.rb +8 -0
  190. data/test/test_switch_node.rb +12 -0
  191. data/test/test_this_node.rb +8 -0
  192. data/test/test_throw_node.rb +7 -0
  193. data/test/test_tokenizer.rb +285 -0
  194. data/test/test_true_node.rb +8 -0
  195. data/test/test_try_node.rb +59 -0
  196. data/test/test_type_of_node.rb +8 -0
  197. data/test/test_unary_minus_node.rb +8 -0
  198. data/test/test_unary_plus_node.rb +8 -0
  199. data/test/test_unsigned_right_shift_node.rb +8 -0
  200. data/test/test_var_decl_node.rb +21 -0
  201. data/test/test_var_statement_node.rb +14 -0
  202. data/test/test_void_node.rb +8 -0
  203. data/test/test_while_node.rb +15 -0
  204. data/test/test_with_node.rb +8 -0
  205. metadata +293 -0
@@ -0,0 +1,34 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class PointcutVisitorTest < Test::Unit::TestCase
4
+ include RKelly::Visitors
5
+ include RKelly::Nodes
6
+
7
+ def setup
8
+ @parser = RKelly::Parser.new
9
+ end
10
+
11
+ def test_visit_NumberNode
12
+ ast = @parser.parse('Element.update(10, 10)')
13
+ assert_equal(2, ast.pointcut('10').matches.length)
14
+ end
15
+
16
+ def test_visit_RegexpNode
17
+ ast = @parser.parse('Element.update(/asdf/, /asdf/)')
18
+ assert_equal(2, ast.pointcut('/asdf/').matches.length)
19
+ end
20
+
21
+ def test_visit_ContinueNode
22
+ ast = @parser.parse('function foo() { continue; }')
23
+ cut = ast.pointcut('continue')
24
+ assert_equal(1, cut.matches.length)
25
+ assert cut.matches.first.is_a?(ContinueNode)
26
+ end
27
+
28
+ def test_try_catch
29
+ ast = @parser.parse('try { Element.update(10, 10); } catch(e) { }')
30
+ assert_equal(1, ast.pointcut('Element.update(10, 10)').matches.length)
31
+ ast = @parser.parse('try { Element.update("foo", "bar"); } catch(e) { }')
32
+ assert_equal(1, ast.pointcut('Element.update(String, String)').matches.length)
33
+ end
34
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class PostfixNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = PostfixNode.new(NumberNode.new(10), '++')
6
+ assert_sexp([:postfix, [:lit, 10], '++'], node)
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class PrefixNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = PrefixNode.new(NumberNode.new(10), '++')
6
+ assert_sexp([:prefix, [:lit, 10], '++'], node)
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class PropertyNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = PropertyNode.new('foo', NumberNode.new(10))
6
+ assert_sexp([:property, :foo, [:lit, 10]], node)
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class RegexpNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = RegexpNode.new('/yay!/')
6
+ assert_sexp [:lit, '/yay!/'], node
7
+ end
8
+ end
@@ -0,0 +1,22 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class ResolveNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = ResolveNode.new('foo')
6
+ assert_sexp [:resolve, 'foo'], node
7
+ end
8
+
9
+ def test_match
10
+ node = ResolveNode.new('foo')
11
+ node2 = ResolveNode.new('foo')
12
+ assert(node =~ node2)
13
+
14
+ assert(node !~ NumberNode.new(10))
15
+ end
16
+
17
+ def test_is_a
18
+ node = ResolveNode.new('foo')
19
+ node3 = ResolveNode.new('String')
20
+ assert(node3 =~ node)
21
+ end
22
+ end
@@ -0,0 +1,11 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class ReturnNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = ReturnNode.new(nil)
6
+ assert_sexp([:return], node)
7
+
8
+ node = ReturnNode.new(NumberNode.new(10))
9
+ assert_sexp([:return, [:lit, 10]], node)
10
+ end
11
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class RightShiftNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = RightShiftNode.new(NumberNode.new(5), NumberNode.new(10))
6
+ assert_sexp([:rshift, [:lit, 5], [:lit, 10]], node)
7
+ end
8
+ end
@@ -0,0 +1,19 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class RKellyTest < Test::Unit::TestCase
4
+ def test_array_access
5
+ assert_sexp(
6
+ [
7
+ [:var,
8
+ [[:var_decl, :a,
9
+ [:assign, [:bracket_access, [:resolve, "foo"], [:lit, 10]]],
10
+ ]]
11
+ ]
12
+ ],
13
+ RKelly.parse('var a = foo[10];'))
14
+ end
15
+
16
+ def assert_sexp(expected, node)
17
+ assert_equal(expected, node.to_sexp)
18
+ end
19
+ end
@@ -0,0 +1,12 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class RuntimeTest < Test::Unit::TestCase
4
+ def setup
5
+ @runtime = RKelly::Runtime.new
6
+ end
7
+
8
+ def test_call_function
9
+ @runtime.execute("function foo(a) { return a + 2; }")
10
+ assert_equal(12, @runtime.call_function("foo", 10))
11
+ end
12
+ end
@@ -0,0 +1,50 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class TestScopeChain < Test::Unit::TestCase
4
+ def setup
5
+ @scope_chain = RKelly::Runtime::ScopeChain.new
6
+ scope_1 = RKelly::JS::Scope.new
7
+ scope_1.properties[:foo] = 1
8
+ scope_2 = RKelly::JS::Scope.new
9
+ scope_2.properties[:bar] = 10
10
+ @scope_chain << scope_1
11
+ @scope_chain << scope_2
12
+ end
13
+
14
+ def test_global_object_in_chain
15
+ assert @scope_chain.has_property?('NaN')
16
+ end
17
+
18
+ def test_has_property
19
+ assert @scope_chain.has_property?(:foo)
20
+ assert @scope_chain.has_property?(:bar)
21
+ assert !@scope_chain.has_property?(:baz)
22
+ @scope_chain.pop
23
+ assert @scope_chain.has_property?(:bar).nil?
24
+ end
25
+
26
+ def test_find_property
27
+ assert_equal(10, @scope_chain[:bar])
28
+ assert_equal(1, @scope_chain[:foo])
29
+ @scope_chain.pop
30
+ assert(!@scope_chain.has_property?(:bar))
31
+ end
32
+
33
+ def test_add_property
34
+ assert !@scope_chain.has_property?(:baz)
35
+ @scope_chain[:baz] = 10
36
+ assert @scope_chain.has_property?(:baz)
37
+ @scope_chain.pop
38
+ assert !@scope_chain.has_property?(:baz)
39
+ end
40
+
41
+ def test_chain_in_block
42
+ assert !@scope_chain.has_property?(:baz)
43
+ @scope_chain.new_scope { |chain|
44
+ chain[:baz] = 10
45
+ assert @scope_chain.has_property?(:baz)
46
+ assert chain.has_property?(:baz)
47
+ }
48
+ assert @scope_chain.has_property?(:baz).nil?
49
+ end
50
+ end
@@ -0,0 +1,10 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class SetterPropertyNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ body = FunctionBodyNode.new(SourceElementsNode.new([]))
6
+ function = FunctionExprNode.new(nil, body)
7
+ node = SetterPropertyNode.new('foo', function)
8
+ assert_sexp([:setter, :foo, [:func_expr, nil, [], [:func_body, []]]], node)
9
+ end
10
+ end
@@ -0,0 +1,9 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class SourceElementListTest < NodeTestCase
4
+ def test_to_sexp
5
+ num = NumberNode.new(10)
6
+ node = SourceElementsNode.new([num, num])
7
+ assert_sexp [[:lit, 10], [:lit, 10]], node
8
+ end
9
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class StrictEqualNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = StrictEqualNode.new(NumberNode.new(5), NumberNode.new(10))
6
+ assert_sexp([:strict_equal, [:lit, 5], [:lit, 10]], node)
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class StringNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = StringNode.new('"asdf"')
6
+ assert_sexp [:str, '"asdf"'], node
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class SubtractNodeTest < NodeTestCase
4
+ def test_subtract
5
+ node = SubtractNode.new(NumberNode.new(5), NumberNode.new(10))
6
+ assert_sexp([:subtract, [:lit, 5], [:lit, 10]], node)
7
+ end
8
+ end
@@ -0,0 +1,12 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class SwitchNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ clause = CaseClauseNode.new( ResolveNode.new('foo'),
6
+ SourceElementsNode.new([ResolveNode.new('bar')]))
7
+ block = CaseBlockNode.new([clause])
8
+ node = SwitchNode.new(ResolveNode.new('o'), block)
9
+ assert_sexp([:switch, [:resolve, 'o'],[:case_block, [[:case, [:resolve, 'foo'], [[:resolve, 'bar']]]]]],
10
+ node)
11
+ end
12
+ end
@@ -0,0 +1,8 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class ThisNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ node = ThisNode.new('this')
6
+ assert_sexp([:this], node)
7
+ end
8
+ end
@@ -0,0 +1,7 @@
1
+ require File.dirname(__FILE__) + "/helper"
2
+
3
+ class ThrowNodeTest < NodeTestCase
4
+ def test_to_sexp
5
+ assert_sexp([:throw, [:lit, 10]], ThrowNode.new(NumberNode.new(10)))
6
+ end
7
+ end
@@ -0,0 +1,285 @@
1
+ # -*- coding: utf-8 -*-
2
+ require File.dirname(__FILE__) + "/helper"
3
+
4
+ class TokenizerTest < Test::Unit::TestCase
5
+ def setup
6
+ @tokenizer = RKelly::Tokenizer.new
7
+ end
8
+
9
+ {
10
+ :space => " ",
11
+ :tab => "\t",
12
+ :form_feed => "\f",
13
+ :vertical_tab => "\v",
14
+ :no_break_space => [0x00A0].pack("U"),
15
+ :ogham_space_mark => [0x1680].pack("U"),
16
+ :en_quad => [0x2000].pack("U"),
17
+ :em_quad => [0x2001].pack("U"),
18
+ :en_space => [0x2002].pack("U"),
19
+ :em_space => [0x2003].pack("U"),
20
+ :three_per_em_space => [0x2004].pack("U"),
21
+ :four_per_em_space => [0x2005].pack("U"),
22
+ :six_per_em_space => [0x2006].pack("U"),
23
+ :figure_space => [0x2007].pack("U"),
24
+ :punctuation_space => [0x2008].pack("U"),
25
+ :thin_space => [0x2009].pack("U"),
26
+ :hair_space => [0x200a].pack("U"),
27
+ :narrow_no_break_space => [0x202f].pack("U"),
28
+ :medium_mathematical_space => [0x205f].pack("U"),
29
+ :ideographic_space => [0x3000].pack("U"),
30
+
31
+ # Line terminators
32
+ :newline => "\n",
33
+ :carriage_return => "\r",
34
+ :line_separator => [0x2028].pack("U"),
35
+ :paragraph_separator => [0x2029].pack("U"),
36
+ }.each do |name, char|
37
+ define_method(:"test_whitespace_#{name}") do
38
+ assert_equal([[:S, char]], @tokenizer.tokenize(char))
39
+ end
40
+ end
41
+
42
+ def assert_tokens(expected, actual)
43
+ assert_equal(expected, actual.select { |x| x[0] != :S })
44
+ end
45
+
46
+ def test_comments
47
+ tokens = @tokenizer.tokenize("/** Fooo */")
48
+ assert_tokens([[:COMMENT, '/** Fooo */']], tokens)
49
+ end
50
+
51
+ def test_string_single_quote
52
+ tokens = @tokenizer.tokenize("foo = 'hello world';")
53
+ assert_tokens([
54
+ [:IDENT, 'foo'],
55
+ ['=', '='],
56
+ [:STRING, "'hello world'"],
57
+ [';', ';'],
58
+ ], tokens)
59
+ end
60
+
61
+ def test_string_double_quote
62
+ tokens = @tokenizer.tokenize('foo = "hello world";')
63
+ assert_tokens([
64
+ [:IDENT, 'foo'],
65
+ ['=', '='],
66
+ [:STRING, '"hello world"'],
67
+ [';', ';'],
68
+ ], tokens)
69
+ end
70
+
71
+ def test_number_parse
72
+ tokens = @tokenizer.tokenize('3.')
73
+ assert_tokens([[:NUMBER, 3.0]], tokens)
74
+
75
+ tokens = @tokenizer.tokenize('3.e1')
76
+ assert_tokens([[:NUMBER, 30]], tokens)
77
+
78
+ tokens = @tokenizer.tokenize('.001')
79
+ assert_tokens([[:NUMBER, 0.001]], tokens)
80
+
81
+ tokens = @tokenizer.tokenize('3.e-1')
82
+ assert_tokens([[:NUMBER, 0.30]], tokens)
83
+ end
84
+
85
+ def test_identifier
86
+ tokens = @tokenizer.tokenize("foo")
87
+ assert_tokens([[:IDENT, 'foo']], tokens)
88
+ end
89
+
90
+ def test_ignore_identifier
91
+ tokens = @tokenizer.tokenize("0foo")
92
+ assert_tokens([[:NUMBER, 0], [:IDENT, 'foo']], tokens)
93
+ end
94
+
95
+ def test_increment
96
+ tokens = @tokenizer.tokenize("foo += 1;")
97
+ assert_tokens([
98
+ [:IDENT, 'foo'],
99
+ [:PLUSEQUAL, '+='],
100
+ [:NUMBER, 1],
101
+ [';', ';'],
102
+ ], tokens)
103
+ end
104
+
105
+ def test_regular_expression
106
+ tokens = @tokenizer.tokenize("foo = /=asdf/;")
107
+ assert_tokens([
108
+ [:IDENT, 'foo'],
109
+ ['=', '='],
110
+ [:REGEXP, '/=asdf/'],
111
+ [';', ';'],
112
+ ], tokens)
113
+ end
114
+
115
+ def test_regular_expression_invalid
116
+ tokens = @tokenizer.tokenize("foo = (1 / 2) / 3")
117
+ assert_tokens([[:IDENT, "foo"],
118
+ ["=", "="],
119
+ ["(", "("],
120
+ [:NUMBER, 1],
121
+ ["/", "/"],
122
+ [:NUMBER, 2],
123
+ [")", ")"],
124
+ ["/", "/"],
125
+ [:NUMBER, 3]
126
+ ], tokens)
127
+ end
128
+
129
+ def test_regular_expression_escape
130
+ tokens = @tokenizer.tokenize('foo = /\/asdf/gi;')
131
+ assert_tokens([
132
+ [:IDENT, 'foo'],
133
+ ['=', '='],
134
+ [:REGEXP, '/\/asdf/gi'],
135
+ [';', ';'],
136
+ ], tokens)
137
+ end
138
+
139
+ def test_regular_expression_with_slash_inside_charset
140
+ tokens = @tokenizer.tokenize('foo = /[/]/;')
141
+ assert_tokens([
142
+ [:IDENT, 'foo'],
143
+ ['=', '='],
144
+ [:REGEXP, '/[/]/'],
145
+ [';', ';'],
146
+ ], tokens)
147
+ end
148
+
149
+ def test_regular_expression_is_not_found_if_prev_token_implies_division
150
+ {:IDENT => 'foo',
151
+ :TRUE => 'true',
152
+ :NUMBER => 1,
153
+ ')' => ')',
154
+ ']' => ']',
155
+ '}' => '}'}.each do |name, value|
156
+ tokens = @tokenizer.tokenize("#{value}/2/3")
157
+ assert_tokens([
158
+ [name, value],
159
+ ["/", "/"],
160
+ [:NUMBER, 2],
161
+ ["/", "/"],
162
+ [:NUMBER, 3],
163
+ ], tokens)
164
+ end
165
+ end
166
+
167
+ def test_regular_expression_is_found_if_prev_token_is_non_literal_keyword
168
+ {:RETURN => 'return',
169
+ :THROW => 'throw'}.each do |name, value|
170
+ tokens = @tokenizer.tokenize("#{value}/2/")
171
+ assert_tokens([
172
+ [name, value],
173
+ [:REGEXP, "/2/"],
174
+ ], tokens)
175
+ end
176
+ end
177
+
178
+ def test_regular_expression_is_not_found_if_block_comment_with_re_modifier
179
+ tokens = @tokenizer.tokenize("/**/i")
180
+ assert_tokens([
181
+ [:COMMENT, "/**/"],
182
+ [:IDENT, "i"]
183
+ ], tokens)
184
+ end
185
+
186
+ def test_comment_assign
187
+ tokens = @tokenizer.tokenize("foo = /**/;")
188
+ assert_tokens([
189
+ [:IDENT, 'foo'],
190
+ ['=', '='],
191
+ [:COMMENT, '/**/'],
192
+ [';', ';'],
193
+ ], tokens)
194
+
195
+ tokens = @tokenizer.tokenize("foo = //;")
196
+ assert_tokens([
197
+ [:IDENT, 'foo'],
198
+ ['=', '='],
199
+ [:COMMENT, '//;'],
200
+ ], tokens)
201
+ end
202
+
203
+ def test_unicode_string
204
+ tokens = @tokenizer.tokenize("foo = 'öäüõ';")
205
+ assert_tokens([
206
+ [:IDENT, 'foo'],
207
+ ['=', '='],
208
+ [:STRING, "'öäüõ'"],
209
+ [';', ';'],
210
+ ], tokens)
211
+ end
212
+
213
+ def test_unicode_regex
214
+ tokens = @tokenizer.tokenize("foo = /öäüõ/;")
215
+ assert_tokens([
216
+ [:IDENT, 'foo'],
217
+ ['=', '='],
218
+ [:REGEXP, "/öäüõ/"],
219
+ [';', ';'],
220
+ ], tokens)
221
+ end
222
+
223
+ %w{
224
+ break case catch continue default delete do else finally for function
225
+ if in instanceof new return switch this throw try typeof var void while
226
+ with
227
+
228
+ const true false null debugger
229
+ }.each do |kw|
230
+ define_method(:"test_keyword_#{kw}") do
231
+ tokens = @tokenizer.tokenize(kw)
232
+ assert_equal 1, tokens.length
233
+ assert_equal([[kw.upcase.to_sym, kw]], tokens)
234
+ end
235
+ end
236
+
237
+ %w{
238
+ class enum extends super export import
239
+ }.each do |rw|
240
+ define_method(:"test_future_reserved_word_#{rw}_is_reserved") do
241
+ tokens = @tokenizer.tokenize(rw)
242
+ assert_equal 1, tokens.length
243
+ assert_equal([[:RESERVED, rw]], tokens)
244
+ end
245
+ end
246
+
247
+ %w{
248
+ implements let private public yield
249
+ interface package protected static
250
+ }.each do |rw|
251
+ define_method(:"test_future_reserved_word_#{rw}_is_identifier") do
252
+ tokens = @tokenizer.tokenize(rw)
253
+ assert_equal 1, tokens.length
254
+ assert_equal([[:IDENT, rw]], tokens)
255
+ end
256
+ end
257
+
258
+ {
259
+ '==' => :EQEQ,
260
+ '!=' => :NE,
261
+ '===' => :STREQ,
262
+ '!==' => :STRNEQ,
263
+ '<=' => :LE,
264
+ '>=' => :GE,
265
+ '||' => :OR,
266
+ '&&' => :AND,
267
+ '++' => :PLUSPLUS,
268
+ '--' => :MINUSMINUS,
269
+ '<<' => :LSHIFT,
270
+ '>>' => :RSHIFT,
271
+ '>>>' => :URSHIFT,
272
+ '+=' => :PLUSEQUAL,
273
+ '-=' => :MINUSEQUAL,
274
+ '*=' => :MULTEQUAL,
275
+ 'null' => :NULL,
276
+ 'true' => :TRUE,
277
+ 'false' => :FALSE,
278
+ }.each do |punctuator, sym|
279
+ define_method(:"test_punctuator_#{sym}") do
280
+ tokens = @tokenizer.tokenize(punctuator)
281
+ assert_equal 1, tokens.length
282
+ assert_equal([[sym, punctuator]], tokens)
283
+ end
284
+ end
285
+ end