prism 0.28.0 → 0.30.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +41 -1
  3. data/CONTRIBUTING.md +0 -4
  4. data/README.md +1 -0
  5. data/config.yml +95 -26
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/ripper_translation.md +22 -0
  8. data/ext/prism/api_node.c +70 -52
  9. data/ext/prism/extconf.rb +27 -23
  10. data/ext/prism/extension.c +107 -372
  11. data/ext/prism/extension.h +1 -1
  12. data/include/prism/ast.h +170 -102
  13. data/include/prism/diagnostic.h +18 -3
  14. data/include/prism/node.h +0 -21
  15. data/include/prism/parser.h +23 -25
  16. data/include/prism/regexp.h +17 -8
  17. data/include/prism/static_literals.h +3 -2
  18. data/include/prism/util/pm_char.h +1 -2
  19. data/include/prism/util/pm_constant_pool.h +0 -8
  20. data/include/prism/util/pm_integer.h +16 -9
  21. data/include/prism/util/pm_string.h +0 -8
  22. data/include/prism/version.h +2 -2
  23. data/include/prism.h +0 -11
  24. data/lib/prism/compiler.rb +3 -0
  25. data/lib/prism/desugar_compiler.rb +4 -4
  26. data/lib/prism/dispatcher.rb +14 -0
  27. data/lib/prism/dot_visitor.rb +54 -35
  28. data/lib/prism/dsl.rb +23 -18
  29. data/lib/prism/ffi.rb +25 -4
  30. data/lib/prism/inspect_visitor.rb +26 -24
  31. data/lib/prism/mutation_compiler.rb +6 -1
  32. data/lib/prism/node.rb +314 -389
  33. data/lib/prism/node_ext.rb +175 -17
  34. data/lib/prism/parse_result/comments.rb +1 -8
  35. data/lib/prism/parse_result/newlines.rb +102 -12
  36. data/lib/prism/parse_result.rb +17 -0
  37. data/lib/prism/reflection.rb +11 -9
  38. data/lib/prism/serialize.rb +91 -68
  39. data/lib/prism/translation/parser/compiler.rb +288 -138
  40. data/lib/prism/translation/parser.rb +7 -2
  41. data/lib/prism/translation/ripper.rb +24 -22
  42. data/lib/prism/translation/ruby_parser.rb +32 -14
  43. data/lib/prism/visitor.rb +3 -0
  44. data/lib/prism.rb +0 -4
  45. data/prism.gemspec +2 -4
  46. data/rbi/prism/node.rbi +114 -57
  47. data/rbi/prism/node_ext.rbi +5 -0
  48. data/rbi/prism/parse_result.rbi +1 -1
  49. data/rbi/prism/visitor.rbi +3 -0
  50. data/rbi/prism.rbi +6 -0
  51. data/sig/prism/dsl.rbs +13 -10
  52. data/sig/prism/lex_compat.rbs +10 -0
  53. data/sig/prism/mutation_compiler.rbs +1 -0
  54. data/sig/prism/node.rbs +72 -48
  55. data/sig/prism/node_ext.rbs +4 -0
  56. data/sig/prism/visitor.rbs +1 -0
  57. data/sig/prism.rbs +21 -0
  58. data/src/diagnostic.c +56 -27
  59. data/src/node.c +432 -1690
  60. data/src/prettyprint.c +97 -54
  61. data/src/prism.c +1286 -1196
  62. data/src/regexp.c +133 -68
  63. data/src/serialize.c +22 -17
  64. data/src/static_literals.c +63 -84
  65. data/src/token_type.c +4 -4
  66. data/src/util/pm_constant_pool.c +0 -8
  67. data/src/util/pm_integer.c +39 -11
  68. data/src/util/pm_string.c +0 -12
  69. data/src/util/pm_strpbrk.c +32 -6
  70. metadata +3 -5
  71. data/include/prism/util/pm_string_list.h +0 -44
  72. data/lib/prism/debug.rb +0 -249
  73. data/src/util/pm_string_list.c +0 -28
data/ext/prism/api_node.c CHANGED
@@ -102,6 +102,7 @@ static VALUE rb_cPrismInterpolatedRegularExpressionNode;
102
102
  static VALUE rb_cPrismInterpolatedStringNode;
103
103
  static VALUE rb_cPrismInterpolatedSymbolNode;
104
104
  static VALUE rb_cPrismInterpolatedXStringNode;
105
+ static VALUE rb_cPrismItLocalVariableReadNode;
105
106
  static VALUE rb_cPrismItParametersNode;
106
107
  static VALUE rb_cPrismKeywordHashNode;
107
108
  static VALUE rb_cPrismKeywordRestParameterNode;
@@ -980,12 +981,6 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
980
981
  pm_node_stack_push(&node_stack, (pm_node_t *) cast->right);
981
982
  break;
982
983
  }
983
- #line 146 "api_node.c.erb"
984
- case PM_RATIONAL_NODE: {
985
- pm_rational_node_t *cast = (pm_rational_node_t *) node;
986
- pm_node_stack_push(&node_stack, (pm_node_t *) cast->numeric);
987
- break;
988
- }
989
984
  #line 146 "api_node.c.erb"
990
985
  case PM_RESCUE_MODIFIER_NODE: {
991
986
  pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node;
@@ -1708,14 +1703,14 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
1708
1703
  assert(cast->write_name != 0);
1709
1704
  argv[6] = RARRAY_AREF(constants, cast->write_name - 1);
1710
1705
 
1711
- // operator
1706
+ // binary_operator
1712
1707
  #line 198 "api_node.c.erb"
1713
- assert(cast->operator != 0);
1714
- argv[7] = RARRAY_AREF(constants, cast->operator - 1);
1708
+ assert(cast->binary_operator != 0);
1709
+ argv[7] = RARRAY_AREF(constants, cast->binary_operator - 1);
1715
1710
 
1716
- // operator_loc
1711
+ // binary_operator_loc
1717
1712
  #line 211 "api_node.c.erb"
1718
- argv[8] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
1713
+ argv[8] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
1719
1714
 
1720
1715
  // value
1721
1716
  #line 186 "api_node.c.erb"
@@ -2009,18 +2004,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
2009
2004
  #line 211 "api_node.c.erb"
2010
2005
  argv[2] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
2011
2006
 
2012
- // operator_loc
2007
+ // binary_operator_loc
2013
2008
  #line 211 "api_node.c.erb"
2014
- argv[3] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
2009
+ argv[3] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
2015
2010
 
2016
2011
  // value
2017
2012
  #line 186 "api_node.c.erb"
2018
2013
  argv[4] = rb_ary_pop(value_stack);
2019
2014
 
2020
- // operator
2015
+ // binary_operator
2021
2016
  #line 198 "api_node.c.erb"
2022
- assert(cast->operator != 0);
2023
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
2017
+ assert(cast->binary_operator != 0);
2018
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
2024
2019
 
2025
2020
  // location
2026
2021
  argv[6] = pm_location_new(parser, node->location.start, node->location.end);
@@ -2176,18 +2171,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
2176
2171
  #line 211 "api_node.c.erb"
2177
2172
  argv[2] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
2178
2173
 
2179
- // operator_loc
2174
+ // binary_operator_loc
2180
2175
  #line 211 "api_node.c.erb"
2181
- argv[3] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
2176
+ argv[3] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
2182
2177
 
2183
2178
  // value
2184
2179
  #line 186 "api_node.c.erb"
2185
2180
  argv[4] = rb_ary_pop(value_stack);
2186
2181
 
2187
- // operator
2182
+ // binary_operator
2188
2183
  #line 198 "api_node.c.erb"
2189
- assert(cast->operator != 0);
2190
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
2184
+ assert(cast->binary_operator != 0);
2185
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
2191
2186
 
2192
2187
  // location
2193
2188
  argv[6] = pm_location_new(parser, node->location.start, node->location.end);
@@ -2293,18 +2288,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
2293
2288
  #line 186 "api_node.c.erb"
2294
2289
  argv[1] = rb_ary_pop(value_stack);
2295
2290
 
2296
- // operator_loc
2291
+ // binary_operator_loc
2297
2292
  #line 211 "api_node.c.erb"
2298
- argv[2] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
2293
+ argv[2] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
2299
2294
 
2300
2295
  // value
2301
2296
  #line 186 "api_node.c.erb"
2302
2297
  argv[3] = rb_ary_pop(value_stack);
2303
2298
 
2304
- // operator
2299
+ // binary_operator
2305
2300
  #line 198 "api_node.c.erb"
2306
- assert(cast->operator != 0);
2307
- argv[4] = RARRAY_AREF(constants, cast->operator - 1);
2301
+ assert(cast->binary_operator != 0);
2302
+ argv[4] = RARRAY_AREF(constants, cast->binary_operator - 1);
2308
2303
 
2309
2304
  // location
2310
2305
  argv[5] = pm_location_new(parser, node->location.start, node->location.end);
@@ -2894,18 +2889,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
2894
2889
  #line 211 "api_node.c.erb"
2895
2890
  argv[2] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
2896
2891
 
2897
- // operator_loc
2892
+ // binary_operator_loc
2898
2893
  #line 211 "api_node.c.erb"
2899
- argv[3] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
2894
+ argv[3] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
2900
2895
 
2901
2896
  // value
2902
2897
  #line 186 "api_node.c.erb"
2903
2898
  argv[4] = rb_ary_pop(value_stack);
2904
2899
 
2905
- // operator
2900
+ // binary_operator
2906
2901
  #line 198 "api_node.c.erb"
2907
- assert(cast->operator != 0);
2908
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
2902
+ assert(cast->binary_operator != 0);
2903
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
2909
2904
 
2910
2905
  // location
2911
2906
  argv[6] = pm_location_new(parser, node->location.start, node->location.end);
@@ -3280,14 +3275,14 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3280
3275
  #line 186 "api_node.c.erb"
3281
3276
  argv[7] = rb_ary_pop(value_stack);
3282
3277
 
3283
- // operator
3278
+ // binary_operator
3284
3279
  #line 198 "api_node.c.erb"
3285
- assert(cast->operator != 0);
3286
- argv[8] = RARRAY_AREF(constants, cast->operator - 1);
3280
+ assert(cast->binary_operator != 0);
3281
+ argv[8] = RARRAY_AREF(constants, cast->binary_operator - 1);
3287
3282
 
3288
- // operator_loc
3283
+ // binary_operator_loc
3289
3284
  #line 211 "api_node.c.erb"
3290
- argv[9] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
3285
+ argv[9] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
3291
3286
 
3292
3287
  // value
3293
3288
  #line 186 "api_node.c.erb"
@@ -3435,18 +3430,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3435
3430
  #line 211 "api_node.c.erb"
3436
3431
  argv[2] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
3437
3432
 
3438
- // operator_loc
3433
+ // binary_operator_loc
3439
3434
  #line 211 "api_node.c.erb"
3440
- argv[3] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
3435
+ argv[3] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
3441
3436
 
3442
3437
  // value
3443
3438
  #line 186 "api_node.c.erb"
3444
3439
  argv[4] = rb_ary_pop(value_stack);
3445
3440
 
3446
- // operator
3441
+ // binary_operator
3447
3442
  #line 198 "api_node.c.erb"
3448
- assert(cast->operator != 0);
3449
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
3443
+ assert(cast->binary_operator != 0);
3444
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
3450
3445
 
3451
3446
  // location
3452
3447
  argv[6] = pm_location_new(parser, node->location.start, node->location.end);
@@ -3733,6 +3728,19 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3733
3728
  rb_ary_push(value_stack, rb_class_new_instance(5, argv, rb_cPrismInterpolatedXStringNode));
3734
3729
  break;
3735
3730
  }
3731
+ #line 172 "api_node.c.erb"
3732
+ case PM_IT_LOCAL_VARIABLE_READ_NODE: {
3733
+ VALUE argv[2];
3734
+
3735
+ // source
3736
+ argv[0] = source;
3737
+
3738
+ // location
3739
+ argv[1] = pm_location_new(parser, node->location.start, node->location.end);
3740
+
3741
+ rb_ary_push(value_stack, rb_class_new_instance(2, argv, rb_cPrismItLocalVariableReadNode));
3742
+ break;
3743
+ }
3736
3744
  #line 172 "api_node.c.erb"
3737
3745
  case PM_IT_PARAMETERS_NODE: {
3738
3746
  VALUE argv[2];
@@ -3889,9 +3897,9 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3889
3897
  #line 211 "api_node.c.erb"
3890
3898
  argv[1] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
3891
3899
 
3892
- // operator_loc
3900
+ // binary_operator_loc
3893
3901
  #line 211 "api_node.c.erb"
3894
- argv[2] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
3902
+ argv[2] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
3895
3903
 
3896
3904
  // value
3897
3905
  #line 186 "api_node.c.erb"
@@ -3902,10 +3910,10 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3902
3910
  assert(cast->name != 0);
3903
3911
  argv[4] = RARRAY_AREF(constants, cast->name - 1);
3904
3912
 
3905
- // operator
3913
+ // binary_operator
3906
3914
  #line 198 "api_node.c.erb"
3907
- assert(cast->operator != 0);
3908
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
3915
+ assert(cast->binary_operator != 0);
3916
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
3909
3917
 
3910
3918
  // depth
3911
3919
  #line 220 "api_node.c.erb"
@@ -4723,19 +4731,28 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
4723
4731
  }
4724
4732
  #line 172 "api_node.c.erb"
4725
4733
  case PM_RATIONAL_NODE: {
4726
- VALUE argv[3];
4734
+ pm_rational_node_t *cast = (pm_rational_node_t *) node;
4735
+ VALUE argv[5];
4727
4736
 
4728
4737
  // source
4729
4738
  argv[0] = source;
4730
4739
 
4731
- // numeric
4732
- #line 186 "api_node.c.erb"
4733
- argv[1] = rb_ary_pop(value_stack);
4740
+ // flags
4741
+ #line 223 "api_node.c.erb"
4742
+ argv[1] = ULONG2NUM(node->flags & ~PM_NODE_FLAG_COMMON_MASK);
4743
+
4744
+ // numerator
4745
+ #line 226 "api_node.c.erb"
4746
+ argv[2] = pm_integer_new(&cast->numerator);
4747
+
4748
+ // denominator
4749
+ #line 226 "api_node.c.erb"
4750
+ argv[3] = pm_integer_new(&cast->denominator);
4734
4751
 
4735
4752
  // location
4736
- argv[2] = pm_location_new(parser, node->location.start, node->location.end);
4753
+ argv[4] = pm_location_new(parser, node->location.start, node->location.end);
4737
4754
 
4738
- rb_ary_push(value_stack, rb_class_new_instance(3, argv, rb_cPrismRationalNode));
4755
+ rb_ary_push(value_stack, rb_class_new_instance(5, argv, rb_cPrismRationalNode));
4739
4756
  break;
4740
4757
  }
4741
4758
  #line 172 "api_node.c.erb"
@@ -5578,6 +5595,7 @@ Init_prism_api_node(void) {
5578
5595
  rb_cPrismInterpolatedStringNode = rb_define_class_under(rb_cPrism, "InterpolatedStringNode", rb_cPrismNode);
5579
5596
  rb_cPrismInterpolatedSymbolNode = rb_define_class_under(rb_cPrism, "InterpolatedSymbolNode", rb_cPrismNode);
5580
5597
  rb_cPrismInterpolatedXStringNode = rb_define_class_under(rb_cPrism, "InterpolatedXStringNode", rb_cPrismNode);
5598
+ rb_cPrismItLocalVariableReadNode = rb_define_class_under(rb_cPrism, "ItLocalVariableReadNode", rb_cPrismNode);
5581
5599
  rb_cPrismItParametersNode = rb_define_class_under(rb_cPrism, "ItParametersNode", rb_cPrismNode);
5582
5600
  rb_cPrismKeywordHashNode = rb_define_class_under(rb_cPrism, "KeywordHashNode", rb_cPrismNode);
5583
5601
  rb_cPrismKeywordRestParameterNode = rb_define_class_under(rb_cPrism, "KeywordRestParameterNode", rb_cPrismNode);
data/ext/prism/extconf.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "rbconfig"
4
+
3
5
  if ARGV.delete("--help")
4
6
  print(<<~TEXT)
5
7
  USAGE: ruby #{$PROGRAM_NAME} [options]
@@ -40,26 +42,6 @@ def generate_templates
40
42
  end
41
43
  end
42
44
 
43
- # We're going to need to run `make` using prism's `Makefile`. We want to match
44
- # up as much of the configuration to the configuration that built the current
45
- # version of Ruby as possible.
46
- require "rbconfig"
47
- env = RbConfig::CONFIG.slice("SOEXT", "CPPFLAGS", "CFLAGS", "CC", "AR", "ARFLAGS", "MAKEDIRS", "RMALL")
48
-
49
- # It's possible that the Ruby that is being run wasn't actually compiled on this
50
- # machine, in which case the configuration might be incorrect. In this case
51
- # we'll need to do some additional checks and potentially fall back to defaults.
52
- if env.key?("CC") && !File.exist?(env["CC"])
53
- env.delete("CC")
54
- env.delete("CFLAGS")
55
- env.delete("CPPFLAGS")
56
- end
57
-
58
- if env.key?("AR") && !File.exist?(env["AR"])
59
- env.delete("AR")
60
- env.delete("ARFLAGS")
61
- end
62
-
63
45
  # Runs `make` in the root directory of the project. Note that this is the
64
46
  # `Makefile` for the overall project, not the `Makefile` that is being generated
65
47
  # by this script.`
@@ -77,15 +59,37 @@ end
77
59
 
78
60
  # On non-CRuby we only need the shared library since we'll interface with it
79
61
  # through FFI, so we'll build only that and not the C extension. We also avoid
80
- # `require "mkmf"` as that prepends the LLVM toolchain to PATH on TruffleRuby,
81
- # but we want to use the native toolchain here since libprism is run natively.
62
+ # `require "mkmf"` as that prepends the GraalVM LLVM toolchain to PATH on TruffleRuby < 24.0,
63
+ # but we want to use the system toolchain here since libprism is run natively.
82
64
  if RUBY_ENGINE != "ruby"
83
65
  generate_templates
84
- make(env, "build/libprism.#{RbConfig::CONFIG["SOEXT"]}")
66
+ soext = RbConfig::CONFIG["SOEXT"]
67
+ # Pass SOEXT to avoid an extra subprocess just to query that
68
+ make({ "SOEXT" => soext }, "build/libprism.#{soext}")
85
69
  File.write("Makefile", "all install clean:\n\t@#{RbConfig::CONFIG["NULLCMD"]}\n")
86
70
  return
87
71
  end
88
72
 
73
+ # We're going to need to run `make` using prism's `Makefile`.
74
+ # We want to use the same toolchain (compiler, flags, etc) to compile libprism.a and
75
+ # the C extension since they will be linked together.
76
+ # The C extension uses RbConfig, which contains values from the toolchain that built the running Ruby.
77
+ env = RbConfig::CONFIG.slice("SOEXT", "CPPFLAGS", "CFLAGS", "CC", "AR", "ARFLAGS", "MAKEDIRS", "RMALL")
78
+
79
+ # It's possible that the Ruby that is being run wasn't actually compiled on this
80
+ # machine, in which case parts of RbConfig might be incorrect. In this case
81
+ # we'll need to do some additional checks and potentially fall back to defaults.
82
+ if env.key?("CC") && !File.exist?(env["CC"])
83
+ env.delete("CC")
84
+ env.delete("CFLAGS")
85
+ env.delete("CPPFLAGS")
86
+ end
87
+
88
+ if env.key?("AR") && !File.exist?(env["AR"])
89
+ env.delete("AR")
90
+ env.delete("ARFLAGS")
91
+ end
92
+
89
93
  require "mkmf"
90
94
 
91
95
  # First, ensure that we can find the header for the prism library.