prism 0.28.0 → 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +41 -1
  3. data/CONTRIBUTING.md +0 -4
  4. data/README.md +1 -0
  5. data/config.yml +95 -26
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/ripper_translation.md +22 -0
  8. data/ext/prism/api_node.c +70 -52
  9. data/ext/prism/extconf.rb +27 -23
  10. data/ext/prism/extension.c +107 -372
  11. data/ext/prism/extension.h +1 -1
  12. data/include/prism/ast.h +170 -102
  13. data/include/prism/diagnostic.h +18 -3
  14. data/include/prism/node.h +0 -21
  15. data/include/prism/parser.h +23 -25
  16. data/include/prism/regexp.h +17 -8
  17. data/include/prism/static_literals.h +3 -2
  18. data/include/prism/util/pm_char.h +1 -2
  19. data/include/prism/util/pm_constant_pool.h +0 -8
  20. data/include/prism/util/pm_integer.h +16 -9
  21. data/include/prism/util/pm_string.h +0 -8
  22. data/include/prism/version.h +2 -2
  23. data/include/prism.h +0 -11
  24. data/lib/prism/compiler.rb +3 -0
  25. data/lib/prism/desugar_compiler.rb +4 -4
  26. data/lib/prism/dispatcher.rb +14 -0
  27. data/lib/prism/dot_visitor.rb +54 -35
  28. data/lib/prism/dsl.rb +23 -18
  29. data/lib/prism/ffi.rb +25 -4
  30. data/lib/prism/inspect_visitor.rb +26 -24
  31. data/lib/prism/mutation_compiler.rb +6 -1
  32. data/lib/prism/node.rb +314 -389
  33. data/lib/prism/node_ext.rb +175 -17
  34. data/lib/prism/parse_result/comments.rb +1 -8
  35. data/lib/prism/parse_result/newlines.rb +102 -12
  36. data/lib/prism/parse_result.rb +17 -0
  37. data/lib/prism/reflection.rb +11 -9
  38. data/lib/prism/serialize.rb +91 -68
  39. data/lib/prism/translation/parser/compiler.rb +288 -138
  40. data/lib/prism/translation/parser.rb +7 -2
  41. data/lib/prism/translation/ripper.rb +24 -22
  42. data/lib/prism/translation/ruby_parser.rb +32 -14
  43. data/lib/prism/visitor.rb +3 -0
  44. data/lib/prism.rb +0 -4
  45. data/prism.gemspec +2 -4
  46. data/rbi/prism/node.rbi +114 -57
  47. data/rbi/prism/node_ext.rbi +5 -0
  48. data/rbi/prism/parse_result.rbi +1 -1
  49. data/rbi/prism/visitor.rbi +3 -0
  50. data/rbi/prism.rbi +6 -0
  51. data/sig/prism/dsl.rbs +13 -10
  52. data/sig/prism/lex_compat.rbs +10 -0
  53. data/sig/prism/mutation_compiler.rbs +1 -0
  54. data/sig/prism/node.rbs +72 -48
  55. data/sig/prism/node_ext.rbs +4 -0
  56. data/sig/prism/visitor.rbs +1 -0
  57. data/sig/prism.rbs +21 -0
  58. data/src/diagnostic.c +56 -27
  59. data/src/node.c +432 -1690
  60. data/src/prettyprint.c +97 -54
  61. data/src/prism.c +1286 -1196
  62. data/src/regexp.c +133 -68
  63. data/src/serialize.c +22 -17
  64. data/src/static_literals.c +63 -84
  65. data/src/token_type.c +4 -4
  66. data/src/util/pm_constant_pool.c +0 -8
  67. data/src/util/pm_integer.c +39 -11
  68. data/src/util/pm_string.c +0 -12
  69. data/src/util/pm_strpbrk.c +32 -6
  70. metadata +3 -5
  71. data/include/prism/util/pm_string_list.h +0 -44
  72. data/lib/prism/debug.rb +0 -249
  73. data/src/util/pm_string_list.c +0 -28
data/ext/prism/api_node.c CHANGED
@@ -102,6 +102,7 @@ static VALUE rb_cPrismInterpolatedRegularExpressionNode;
102
102
  static VALUE rb_cPrismInterpolatedStringNode;
103
103
  static VALUE rb_cPrismInterpolatedSymbolNode;
104
104
  static VALUE rb_cPrismInterpolatedXStringNode;
105
+ static VALUE rb_cPrismItLocalVariableReadNode;
105
106
  static VALUE rb_cPrismItParametersNode;
106
107
  static VALUE rb_cPrismKeywordHashNode;
107
108
  static VALUE rb_cPrismKeywordRestParameterNode;
@@ -980,12 +981,6 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
980
981
  pm_node_stack_push(&node_stack, (pm_node_t *) cast->right);
981
982
  break;
982
983
  }
983
- #line 146 "api_node.c.erb"
984
- case PM_RATIONAL_NODE: {
985
- pm_rational_node_t *cast = (pm_rational_node_t *) node;
986
- pm_node_stack_push(&node_stack, (pm_node_t *) cast->numeric);
987
- break;
988
- }
989
984
  #line 146 "api_node.c.erb"
990
985
  case PM_RESCUE_MODIFIER_NODE: {
991
986
  pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node;
@@ -1708,14 +1703,14 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
1708
1703
  assert(cast->write_name != 0);
1709
1704
  argv[6] = RARRAY_AREF(constants, cast->write_name - 1);
1710
1705
 
1711
- // operator
1706
+ // binary_operator
1712
1707
  #line 198 "api_node.c.erb"
1713
- assert(cast->operator != 0);
1714
- argv[7] = RARRAY_AREF(constants, cast->operator - 1);
1708
+ assert(cast->binary_operator != 0);
1709
+ argv[7] = RARRAY_AREF(constants, cast->binary_operator - 1);
1715
1710
 
1716
- // operator_loc
1711
+ // binary_operator_loc
1717
1712
  #line 211 "api_node.c.erb"
1718
- argv[8] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
1713
+ argv[8] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
1719
1714
 
1720
1715
  // value
1721
1716
  #line 186 "api_node.c.erb"
@@ -2009,18 +2004,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
2009
2004
  #line 211 "api_node.c.erb"
2010
2005
  argv[2] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
2011
2006
 
2012
- // operator_loc
2007
+ // binary_operator_loc
2013
2008
  #line 211 "api_node.c.erb"
2014
- argv[3] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
2009
+ argv[3] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
2015
2010
 
2016
2011
  // value
2017
2012
  #line 186 "api_node.c.erb"
2018
2013
  argv[4] = rb_ary_pop(value_stack);
2019
2014
 
2020
- // operator
2015
+ // binary_operator
2021
2016
  #line 198 "api_node.c.erb"
2022
- assert(cast->operator != 0);
2023
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
2017
+ assert(cast->binary_operator != 0);
2018
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
2024
2019
 
2025
2020
  // location
2026
2021
  argv[6] = pm_location_new(parser, node->location.start, node->location.end);
@@ -2176,18 +2171,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
2176
2171
  #line 211 "api_node.c.erb"
2177
2172
  argv[2] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
2178
2173
 
2179
- // operator_loc
2174
+ // binary_operator_loc
2180
2175
  #line 211 "api_node.c.erb"
2181
- argv[3] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
2176
+ argv[3] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
2182
2177
 
2183
2178
  // value
2184
2179
  #line 186 "api_node.c.erb"
2185
2180
  argv[4] = rb_ary_pop(value_stack);
2186
2181
 
2187
- // operator
2182
+ // binary_operator
2188
2183
  #line 198 "api_node.c.erb"
2189
- assert(cast->operator != 0);
2190
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
2184
+ assert(cast->binary_operator != 0);
2185
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
2191
2186
 
2192
2187
  // location
2193
2188
  argv[6] = pm_location_new(parser, node->location.start, node->location.end);
@@ -2293,18 +2288,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
2293
2288
  #line 186 "api_node.c.erb"
2294
2289
  argv[1] = rb_ary_pop(value_stack);
2295
2290
 
2296
- // operator_loc
2291
+ // binary_operator_loc
2297
2292
  #line 211 "api_node.c.erb"
2298
- argv[2] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
2293
+ argv[2] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
2299
2294
 
2300
2295
  // value
2301
2296
  #line 186 "api_node.c.erb"
2302
2297
  argv[3] = rb_ary_pop(value_stack);
2303
2298
 
2304
- // operator
2299
+ // binary_operator
2305
2300
  #line 198 "api_node.c.erb"
2306
- assert(cast->operator != 0);
2307
- argv[4] = RARRAY_AREF(constants, cast->operator - 1);
2301
+ assert(cast->binary_operator != 0);
2302
+ argv[4] = RARRAY_AREF(constants, cast->binary_operator - 1);
2308
2303
 
2309
2304
  // location
2310
2305
  argv[5] = pm_location_new(parser, node->location.start, node->location.end);
@@ -2894,18 +2889,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
2894
2889
  #line 211 "api_node.c.erb"
2895
2890
  argv[2] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
2896
2891
 
2897
- // operator_loc
2892
+ // binary_operator_loc
2898
2893
  #line 211 "api_node.c.erb"
2899
- argv[3] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
2894
+ argv[3] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
2900
2895
 
2901
2896
  // value
2902
2897
  #line 186 "api_node.c.erb"
2903
2898
  argv[4] = rb_ary_pop(value_stack);
2904
2899
 
2905
- // operator
2900
+ // binary_operator
2906
2901
  #line 198 "api_node.c.erb"
2907
- assert(cast->operator != 0);
2908
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
2902
+ assert(cast->binary_operator != 0);
2903
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
2909
2904
 
2910
2905
  // location
2911
2906
  argv[6] = pm_location_new(parser, node->location.start, node->location.end);
@@ -3280,14 +3275,14 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3280
3275
  #line 186 "api_node.c.erb"
3281
3276
  argv[7] = rb_ary_pop(value_stack);
3282
3277
 
3283
- // operator
3278
+ // binary_operator
3284
3279
  #line 198 "api_node.c.erb"
3285
- assert(cast->operator != 0);
3286
- argv[8] = RARRAY_AREF(constants, cast->operator - 1);
3280
+ assert(cast->binary_operator != 0);
3281
+ argv[8] = RARRAY_AREF(constants, cast->binary_operator - 1);
3287
3282
 
3288
- // operator_loc
3283
+ // binary_operator_loc
3289
3284
  #line 211 "api_node.c.erb"
3290
- argv[9] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
3285
+ argv[9] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
3291
3286
 
3292
3287
  // value
3293
3288
  #line 186 "api_node.c.erb"
@@ -3435,18 +3430,18 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3435
3430
  #line 211 "api_node.c.erb"
3436
3431
  argv[2] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
3437
3432
 
3438
- // operator_loc
3433
+ // binary_operator_loc
3439
3434
  #line 211 "api_node.c.erb"
3440
- argv[3] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
3435
+ argv[3] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
3441
3436
 
3442
3437
  // value
3443
3438
  #line 186 "api_node.c.erb"
3444
3439
  argv[4] = rb_ary_pop(value_stack);
3445
3440
 
3446
- // operator
3441
+ // binary_operator
3447
3442
  #line 198 "api_node.c.erb"
3448
- assert(cast->operator != 0);
3449
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
3443
+ assert(cast->binary_operator != 0);
3444
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
3450
3445
 
3451
3446
  // location
3452
3447
  argv[6] = pm_location_new(parser, node->location.start, node->location.end);
@@ -3733,6 +3728,19 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3733
3728
  rb_ary_push(value_stack, rb_class_new_instance(5, argv, rb_cPrismInterpolatedXStringNode));
3734
3729
  break;
3735
3730
  }
3731
+ #line 172 "api_node.c.erb"
3732
+ case PM_IT_LOCAL_VARIABLE_READ_NODE: {
3733
+ VALUE argv[2];
3734
+
3735
+ // source
3736
+ argv[0] = source;
3737
+
3738
+ // location
3739
+ argv[1] = pm_location_new(parser, node->location.start, node->location.end);
3740
+
3741
+ rb_ary_push(value_stack, rb_class_new_instance(2, argv, rb_cPrismItLocalVariableReadNode));
3742
+ break;
3743
+ }
3736
3744
  #line 172 "api_node.c.erb"
3737
3745
  case PM_IT_PARAMETERS_NODE: {
3738
3746
  VALUE argv[2];
@@ -3889,9 +3897,9 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3889
3897
  #line 211 "api_node.c.erb"
3890
3898
  argv[1] = pm_location_new(parser, cast->name_loc.start, cast->name_loc.end);
3891
3899
 
3892
- // operator_loc
3900
+ // binary_operator_loc
3893
3901
  #line 211 "api_node.c.erb"
3894
- argv[2] = pm_location_new(parser, cast->operator_loc.start, cast->operator_loc.end);
3902
+ argv[2] = pm_location_new(parser, cast->binary_operator_loc.start, cast->binary_operator_loc.end);
3895
3903
 
3896
3904
  // value
3897
3905
  #line 186 "api_node.c.erb"
@@ -3902,10 +3910,10 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
3902
3910
  assert(cast->name != 0);
3903
3911
  argv[4] = RARRAY_AREF(constants, cast->name - 1);
3904
3912
 
3905
- // operator
3913
+ // binary_operator
3906
3914
  #line 198 "api_node.c.erb"
3907
- assert(cast->operator != 0);
3908
- argv[5] = RARRAY_AREF(constants, cast->operator - 1);
3915
+ assert(cast->binary_operator != 0);
3916
+ argv[5] = RARRAY_AREF(constants, cast->binary_operator - 1);
3909
3917
 
3910
3918
  // depth
3911
3919
  #line 220 "api_node.c.erb"
@@ -4723,19 +4731,28 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
4723
4731
  }
4724
4732
  #line 172 "api_node.c.erb"
4725
4733
  case PM_RATIONAL_NODE: {
4726
- VALUE argv[3];
4734
+ pm_rational_node_t *cast = (pm_rational_node_t *) node;
4735
+ VALUE argv[5];
4727
4736
 
4728
4737
  // source
4729
4738
  argv[0] = source;
4730
4739
 
4731
- // numeric
4732
- #line 186 "api_node.c.erb"
4733
- argv[1] = rb_ary_pop(value_stack);
4740
+ // flags
4741
+ #line 223 "api_node.c.erb"
4742
+ argv[1] = ULONG2NUM(node->flags & ~PM_NODE_FLAG_COMMON_MASK);
4743
+
4744
+ // numerator
4745
+ #line 226 "api_node.c.erb"
4746
+ argv[2] = pm_integer_new(&cast->numerator);
4747
+
4748
+ // denominator
4749
+ #line 226 "api_node.c.erb"
4750
+ argv[3] = pm_integer_new(&cast->denominator);
4734
4751
 
4735
4752
  // location
4736
- argv[2] = pm_location_new(parser, node->location.start, node->location.end);
4753
+ argv[4] = pm_location_new(parser, node->location.start, node->location.end);
4737
4754
 
4738
- rb_ary_push(value_stack, rb_class_new_instance(3, argv, rb_cPrismRationalNode));
4755
+ rb_ary_push(value_stack, rb_class_new_instance(5, argv, rb_cPrismRationalNode));
4739
4756
  break;
4740
4757
  }
4741
4758
  #line 172 "api_node.c.erb"
@@ -5578,6 +5595,7 @@ Init_prism_api_node(void) {
5578
5595
  rb_cPrismInterpolatedStringNode = rb_define_class_under(rb_cPrism, "InterpolatedStringNode", rb_cPrismNode);
5579
5596
  rb_cPrismInterpolatedSymbolNode = rb_define_class_under(rb_cPrism, "InterpolatedSymbolNode", rb_cPrismNode);
5580
5597
  rb_cPrismInterpolatedXStringNode = rb_define_class_under(rb_cPrism, "InterpolatedXStringNode", rb_cPrismNode);
5598
+ rb_cPrismItLocalVariableReadNode = rb_define_class_under(rb_cPrism, "ItLocalVariableReadNode", rb_cPrismNode);
5581
5599
  rb_cPrismItParametersNode = rb_define_class_under(rb_cPrism, "ItParametersNode", rb_cPrismNode);
5582
5600
  rb_cPrismKeywordHashNode = rb_define_class_under(rb_cPrism, "KeywordHashNode", rb_cPrismNode);
5583
5601
  rb_cPrismKeywordRestParameterNode = rb_define_class_under(rb_cPrism, "KeywordRestParameterNode", rb_cPrismNode);
data/ext/prism/extconf.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "rbconfig"
4
+
3
5
  if ARGV.delete("--help")
4
6
  print(<<~TEXT)
5
7
  USAGE: ruby #{$PROGRAM_NAME} [options]
@@ -40,26 +42,6 @@ def generate_templates
40
42
  end
41
43
  end
42
44
 
43
- # We're going to need to run `make` using prism's `Makefile`. We want to match
44
- # up as much of the configuration to the configuration that built the current
45
- # version of Ruby as possible.
46
- require "rbconfig"
47
- env = RbConfig::CONFIG.slice("SOEXT", "CPPFLAGS", "CFLAGS", "CC", "AR", "ARFLAGS", "MAKEDIRS", "RMALL")
48
-
49
- # It's possible that the Ruby that is being run wasn't actually compiled on this
50
- # machine, in which case the configuration might be incorrect. In this case
51
- # we'll need to do some additional checks and potentially fall back to defaults.
52
- if env.key?("CC") && !File.exist?(env["CC"])
53
- env.delete("CC")
54
- env.delete("CFLAGS")
55
- env.delete("CPPFLAGS")
56
- end
57
-
58
- if env.key?("AR") && !File.exist?(env["AR"])
59
- env.delete("AR")
60
- env.delete("ARFLAGS")
61
- end
62
-
63
45
  # Runs `make` in the root directory of the project. Note that this is the
64
46
  # `Makefile` for the overall project, not the `Makefile` that is being generated
65
47
  # by this script.`
@@ -77,15 +59,37 @@ end
77
59
 
78
60
  # On non-CRuby we only need the shared library since we'll interface with it
79
61
  # through FFI, so we'll build only that and not the C extension. We also avoid
80
- # `require "mkmf"` as that prepends the LLVM toolchain to PATH on TruffleRuby,
81
- # but we want to use the native toolchain here since libprism is run natively.
62
+ # `require "mkmf"` as that prepends the GraalVM LLVM toolchain to PATH on TruffleRuby < 24.0,
63
+ # but we want to use the system toolchain here since libprism is run natively.
82
64
  if RUBY_ENGINE != "ruby"
83
65
  generate_templates
84
- make(env, "build/libprism.#{RbConfig::CONFIG["SOEXT"]}")
66
+ soext = RbConfig::CONFIG["SOEXT"]
67
+ # Pass SOEXT to avoid an extra subprocess just to query that
68
+ make({ "SOEXT" => soext }, "build/libprism.#{soext}")
85
69
  File.write("Makefile", "all install clean:\n\t@#{RbConfig::CONFIG["NULLCMD"]}\n")
86
70
  return
87
71
  end
88
72
 
73
+ # We're going to need to run `make` using prism's `Makefile`.
74
+ # We want to use the same toolchain (compiler, flags, etc) to compile libprism.a and
75
+ # the C extension since they will be linked together.
76
+ # The C extension uses RbConfig, which contains values from the toolchain that built the running Ruby.
77
+ env = RbConfig::CONFIG.slice("SOEXT", "CPPFLAGS", "CFLAGS", "CC", "AR", "ARFLAGS", "MAKEDIRS", "RMALL")
78
+
79
+ # It's possible that the Ruby that is being run wasn't actually compiled on this
80
+ # machine, in which case parts of RbConfig might be incorrect. In this case
81
+ # we'll need to do some additional checks and potentially fall back to defaults.
82
+ if env.key?("CC") && !File.exist?(env["CC"])
83
+ env.delete("CC")
84
+ env.delete("CFLAGS")
85
+ env.delete("CPPFLAGS")
86
+ end
87
+
88
+ if env.key?("AR") && !File.exist?(env["AR"])
89
+ env.delete("AR")
90
+ env.delete("ARFLAGS")
91
+ end
92
+
89
93
  require "mkmf"
90
94
 
91
95
  # First, ensure that we can find the header for the prism library.