prism 0.29.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +115 -1
  3. data/CONTRIBUTING.md +0 -4
  4. data/Makefile +1 -1
  5. data/README.md +4 -0
  6. data/config.yml +920 -148
  7. data/docs/build_system.md +8 -11
  8. data/docs/fuzzing.md +1 -1
  9. data/docs/parsing_rules.md +4 -1
  10. data/docs/relocation.md +34 -0
  11. data/docs/ripper_translation.md +22 -0
  12. data/docs/serialization.md +3 -0
  13. data/ext/prism/api_node.c +2863 -2079
  14. data/ext/prism/extconf.rb +14 -37
  15. data/ext/prism/extension.c +241 -391
  16. data/ext/prism/extension.h +2 -2
  17. data/include/prism/ast.h +2156 -453
  18. data/include/prism/defines.h +58 -7
  19. data/include/prism/diagnostic.h +24 -6
  20. data/include/prism/node.h +0 -21
  21. data/include/prism/options.h +94 -3
  22. data/include/prism/parser.h +82 -40
  23. data/include/prism/regexp.h +18 -8
  24. data/include/prism/static_literals.h +3 -2
  25. data/include/prism/util/pm_char.h +1 -2
  26. data/include/prism/util/pm_constant_pool.h +0 -8
  27. data/include/prism/util/pm_integer.h +22 -15
  28. data/include/prism/util/pm_newline_list.h +11 -0
  29. data/include/prism/util/pm_string.h +28 -12
  30. data/include/prism/version.h +3 -3
  31. data/include/prism.h +47 -11
  32. data/lib/prism/compiler.rb +3 -0
  33. data/lib/prism/desugar_compiler.rb +111 -74
  34. data/lib/prism/dispatcher.rb +16 -1
  35. data/lib/prism/dot_visitor.rb +55 -34
  36. data/lib/prism/dsl.rb +660 -468
  37. data/lib/prism/ffi.rb +113 -8
  38. data/lib/prism/inspect_visitor.rb +296 -64
  39. data/lib/prism/lex_compat.rb +1 -1
  40. data/lib/prism/mutation_compiler.rb +11 -6
  41. data/lib/prism/node.rb +4262 -5023
  42. data/lib/prism/node_ext.rb +91 -14
  43. data/lib/prism/parse_result/comments.rb +0 -7
  44. data/lib/prism/parse_result/errors.rb +65 -0
  45. data/lib/prism/parse_result/newlines.rb +101 -11
  46. data/lib/prism/parse_result.rb +183 -6
  47. data/lib/prism/reflection.rb +12 -10
  48. data/lib/prism/relocation.rb +504 -0
  49. data/lib/prism/serialize.rb +496 -609
  50. data/lib/prism/string_query.rb +30 -0
  51. data/lib/prism/translation/parser/compiler.rb +185 -155
  52. data/lib/prism/translation/parser/lexer.rb +26 -4
  53. data/lib/prism/translation/parser.rb +9 -4
  54. data/lib/prism/translation/ripper.rb +23 -25
  55. data/lib/prism/translation/ruby_parser.rb +86 -17
  56. data/lib/prism/visitor.rb +3 -0
  57. data/lib/prism.rb +6 -8
  58. data/prism.gemspec +9 -5
  59. data/rbi/prism/dsl.rbi +521 -0
  60. data/rbi/prism/node.rbi +1115 -1120
  61. data/rbi/prism/parse_result.rbi +29 -0
  62. data/rbi/prism/string_query.rbi +12 -0
  63. data/rbi/prism/visitor.rbi +3 -0
  64. data/rbi/prism.rbi +36 -30
  65. data/sig/prism/dsl.rbs +190 -303
  66. data/sig/prism/mutation_compiler.rbs +1 -0
  67. data/sig/prism/node.rbs +678 -632
  68. data/sig/prism/parse_result.rbs +22 -0
  69. data/sig/prism/relocation.rbs +185 -0
  70. data/sig/prism/string_query.rbs +11 -0
  71. data/sig/prism/visitor.rbs +1 -0
  72. data/sig/prism.rbs +103 -64
  73. data/src/diagnostic.c +64 -28
  74. data/src/node.c +502 -1739
  75. data/src/options.c +76 -27
  76. data/src/prettyprint.c +188 -112
  77. data/src/prism.c +3376 -2293
  78. data/src/regexp.c +208 -71
  79. data/src/serialize.c +182 -50
  80. data/src/static_literals.c +64 -85
  81. data/src/token_type.c +4 -4
  82. data/src/util/pm_char.c +1 -1
  83. data/src/util/pm_constant_pool.c +0 -8
  84. data/src/util/pm_integer.c +53 -25
  85. data/src/util/pm_newline_list.c +29 -0
  86. data/src/util/pm_string.c +131 -80
  87. data/src/util/pm_strpbrk.c +32 -6
  88. metadata +11 -7
  89. data/include/prism/util/pm_string_list.h +0 -44
  90. data/lib/prism/debug.rb +0 -249
  91. data/lib/prism/translation/parser/rubocop.rb +0 -73
  92. data/src/util/pm_string_list.c +0 -28
data/lib/prism/debug.rb DELETED
@@ -1,249 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Prism
4
- # This module is used for testing and debugging and is not meant to be used by
5
- # consumers of this library.
6
- module Debug
7
- # A wrapper around a RubyVM::InstructionSequence that provides a more
8
- # convenient interface for accessing parts of the iseq.
9
- class ISeq # :nodoc:
10
- attr_reader :parts
11
-
12
- def initialize(parts)
13
- @parts = parts
14
- end
15
-
16
- def type
17
- parts[0]
18
- end
19
-
20
- def local_table
21
- parts[10]
22
- end
23
-
24
- def instructions
25
- parts[13]
26
- end
27
-
28
- def each_child
29
- instructions.each do |instruction|
30
- # Only look at arrays. Other instructions are line numbers or
31
- # tracepoint events.
32
- next unless instruction.is_a?(Array)
33
-
34
- instruction.each do |opnd|
35
- # Only look at arrays. Other operands are literals.
36
- next unless opnd.is_a?(Array)
37
-
38
- # Only look at instruction sequences. Other operands are literals.
39
- next unless opnd[0] == "YARVInstructionSequence/SimpleDataFormat"
40
-
41
- yield ISeq.new(opnd)
42
- end
43
- end
44
- end
45
- end
46
-
47
- private_constant :ISeq
48
-
49
- # :call-seq:
50
- # Debug::cruby_locals(source) -> Array
51
- #
52
- # For the given source, compiles with CRuby and returns a list of all of the
53
- # sets of local variables that were encountered.
54
- def self.cruby_locals(source)
55
- verbose, $VERBOSE = $VERBOSE, nil
56
-
57
- begin
58
- locals = [] #: Array[Array[Symbol | Integer]]
59
- stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)]
60
-
61
- while (iseq = stack.pop)
62
- names = [*iseq.local_table]
63
- names.map!.with_index do |name, index|
64
- # When an anonymous local variable is present in the iseq's local
65
- # table, it is represented as the stack offset from the top.
66
- # However, when these are dumped to binary and read back in, they
67
- # are replaced with the symbol :#arg_rest. To consistently handle
68
- # this, we replace them here with their index.
69
- if name == :"#arg_rest"
70
- names.length - index + 1
71
- else
72
- name
73
- end
74
- end
75
-
76
- locals << names
77
- iseq.each_child { |child| stack << child }
78
- end
79
-
80
- locals
81
- ensure
82
- $VERBOSE = verbose
83
- end
84
- end
85
-
86
- # Used to hold the place of a local that will be in the local table but
87
- # cannot be accessed directly from the source code. For example, the
88
- # iteration variable in a for loop or the positional parameter on a method
89
- # definition that is destructured.
90
- AnonymousLocal = Object.new
91
- private_constant :AnonymousLocal
92
-
93
- # :call-seq:
94
- # Debug::prism_locals(source) -> Array
95
- #
96
- # For the given source, parses with prism and returns a list of all of the
97
- # sets of local variables that were encountered.
98
- def self.prism_locals(source)
99
- locals = [] #: Array[Array[Symbol | Integer]]
100
- stack = [Prism.parse(source).value] #: Array[Prism::node]
101
-
102
- while (node = stack.pop)
103
- case node
104
- when BlockNode, DefNode, LambdaNode
105
- names = node.locals
106
- params =
107
- if node.is_a?(DefNode)
108
- node.parameters
109
- elsif node.parameters.is_a?(NumberedParametersNode)
110
- nil
111
- else
112
- node.parameters&.parameters
113
- end
114
-
115
- # prism places parameters in the same order that they appear in the
116
- # source. CRuby places them in the order that they need to appear
117
- # according to their own internal calling convention. We mimic that
118
- # order here so that we can compare properly.
119
- if params
120
- sorted = [
121
- *params.requireds.map do |required|
122
- if required.is_a?(RequiredParameterNode)
123
- required.name
124
- else
125
- AnonymousLocal
126
- end
127
- end,
128
- *params.optionals.map(&:name),
129
- *((params.rest.name || :*) if params.rest && !params.rest.is_a?(ImplicitRestNode)),
130
- *params.posts.map do |post|
131
- if post.is_a?(RequiredParameterNode)
132
- post.name
133
- else
134
- AnonymousLocal
135
- end
136
- end,
137
- *params.keywords.grep(RequiredKeywordParameterNode).map(&:name),
138
- *params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
139
- ]
140
-
141
- sorted << AnonymousLocal if params.keywords.any?
142
-
143
- if params.keyword_rest.is_a?(ForwardingParameterNode)
144
- sorted.push(:*, :**, :&, :"...")
145
- elsif params.keyword_rest.is_a?(KeywordRestParameterNode)
146
- sorted << (params.keyword_rest.name || :**)
147
- end
148
-
149
- # Recurse down the parameter tree to find any destructured
150
- # parameters and add them after the other parameters.
151
- param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse
152
- while (param = param_stack.pop)
153
- case param
154
- when MultiTargetNode
155
- param_stack.concat(param.rights.reverse)
156
- param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name)
157
- param_stack.concat(param.lefts.reverse)
158
- when RequiredParameterNode
159
- sorted << param.name
160
- when SplatNode
161
- sorted << param.expression.name
162
- end
163
- end
164
-
165
- if params.block
166
- sorted << (params.block.name || :&)
167
- end
168
-
169
- names = sorted.concat(names - sorted)
170
- end
171
-
172
- names.map!.with_index do |name, index|
173
- if name == AnonymousLocal
174
- names.length - index + 1
175
- else
176
- name
177
- end
178
- end
179
-
180
- locals << names
181
- when ClassNode, ModuleNode, ProgramNode, SingletonClassNode
182
- locals << node.locals
183
- when ForNode
184
- locals << [2]
185
- when PostExecutionNode
186
- locals.push([], [])
187
- when InterpolatedRegularExpressionNode
188
- locals << [] if node.once?
189
- end
190
-
191
- stack.concat(node.compact_child_nodes)
192
- end
193
-
194
- locals
195
- end
196
-
197
- # :call-seq:
198
- # Debug::newlines(source) -> Array
199
- #
200
- # For the given source string, return the byte offsets of every newline in
201
- # the source.
202
- def self.newlines(source)
203
- Prism.parse(source).source.offsets
204
- end
205
-
206
- # A wrapping around prism's internal encoding data structures. This is used
207
- # for reflection and debugging purposes.
208
- class Encoding
209
- # The name of the encoding, that can be passed to Encoding.find.
210
- attr_reader :name
211
-
212
- # Initialize a new encoding with the given name and whether or not it is
213
- # a multibyte encoding.
214
- def initialize(name, multibyte)
215
- @name = name
216
- @multibyte = multibyte
217
- end
218
-
219
- # Whether or not the encoding is a multibyte encoding.
220
- def multibyte?
221
- @multibyte
222
- end
223
-
224
- # Returns the number of bytes of the first character in the source string,
225
- # if it is valid for the encoding. Otherwise, returns 0.
226
- def width(source)
227
- Encoding._width(name, source)
228
- end
229
-
230
- # Returns true if the first character in the source string is a valid
231
- # alphanumeric character for the encoding.
232
- def alnum?(source)
233
- Encoding._alnum?(name, source)
234
- end
235
-
236
- # Returns true if the first character in the source string is a valid
237
- # alphabetic character for the encoding.
238
- def alpha?(source)
239
- Encoding._alpha?(name, source)
240
- end
241
-
242
- # Returns true if the first character in the source string is a valid
243
- # uppercase character for the encoding.
244
- def upper?(source)
245
- Encoding._upper?(name, source)
246
- end
247
- end
248
- end
249
- end
@@ -1,73 +0,0 @@
1
- # frozen_string_literal: true
2
- # typed: ignore
3
-
4
- warn "WARN: Prism is directly supported since RuboCop 1.62. The `prism/translation/parser/rubocop` file is deprecated."
5
-
6
- require "parser"
7
- require "rubocop"
8
-
9
- require_relative "../../prism"
10
- require_relative "../parser"
11
-
12
- module Prism
13
- module Translation
14
- class Parser
15
- # This is the special version numbers that should be used in RuboCop
16
- # configuration files to trigger using prism.
17
-
18
- # For Ruby 3.3
19
- VERSION_3_3 = 80_82_73_83_77.33
20
-
21
- # For Ruby 3.4
22
- VERSION_3_4 = 80_82_73_83_77.34
23
-
24
- # This module gets prepended into RuboCop::AST::ProcessedSource.
25
- module ProcessedSource
26
- # This condition is compatible with rubocop-ast versions up to 1.30.0.
27
- if RuboCop::AST::ProcessedSource.instance_method(:parser_class).arity == 1
28
- # Redefine parser_class so that we can inject the prism parser into the
29
- # list of known parsers.
30
- def parser_class(ruby_version)
31
- if ruby_version == Prism::Translation::Parser::VERSION_3_3
32
- warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \
33
- "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead."
34
- require_relative "../parser33"
35
- Prism::Translation::Parser33
36
- elsif ruby_version == Prism::Translation::Parser::VERSION_3_4
37
- warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \
38
- "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead."
39
- require_relative "../parser34"
40
- Prism::Translation::Parser34
41
- else
42
- super
43
- end
44
- end
45
- else
46
- # Redefine parser_class so that we can inject the prism parser into the
47
- # list of known parsers.
48
- def parser_class(ruby_version, _parser_engine)
49
- if ruby_version == Prism::Translation::Parser::VERSION_3_3
50
- warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \
51
- "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead."
52
- require_relative "../parser33"
53
- Prism::Translation::Parser33
54
- elsif ruby_version == Prism::Translation::Parser::VERSION_3_4
55
- warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \
56
- "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead."
57
- require_relative "../parser34"
58
- Prism::Translation::Parser34
59
- else
60
- super
61
- end
62
- end
63
- end
64
- end
65
- end
66
- end
67
- end
68
-
69
- # :stopdoc:
70
- RuboCop::AST::ProcessedSource.prepend(Prism::Translation::Parser::ProcessedSource)
71
- known_rubies = RuboCop::TargetRuby.const_get(:KNOWN_RUBIES)
72
- RuboCop::TargetRuby.send(:remove_const, :KNOWN_RUBIES)
73
- RuboCop::TargetRuby::KNOWN_RUBIES = [*known_rubies, Prism::Translation::Parser::VERSION_3_3].freeze
@@ -1,28 +0,0 @@
1
- #include "prism/util/pm_string_list.h"
2
-
3
- /**
4
- * Append a pm_string_t to the given string list.
5
- */
6
- void
7
- pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string) {
8
- if (string_list->length + 1 > string_list->capacity) {
9
- if (string_list->capacity == 0) {
10
- string_list->capacity = 1;
11
- } else {
12
- string_list->capacity *= 2;
13
- }
14
-
15
- string_list->strings = xrealloc(string_list->strings, string_list->capacity * sizeof(pm_string_t));
16
- if (string_list->strings == NULL) abort();
17
- }
18
-
19
- string_list->strings[string_list->length++] = *string;
20
- }
21
-
22
- /**
23
- * Free the memory associated with the string list
24
- */
25
- void
26
- pm_string_list_free(pm_string_list_t *string_list) {
27
- xfree(string_list->strings);
28
- }