prism 0.29.0 → 1.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +115 -1
  3. data/CONTRIBUTING.md +0 -4
  4. data/Makefile +1 -1
  5. data/README.md +4 -0
  6. data/config.yml +920 -148
  7. data/docs/build_system.md +8 -11
  8. data/docs/fuzzing.md +1 -1
  9. data/docs/parsing_rules.md +4 -1
  10. data/docs/relocation.md +34 -0
  11. data/docs/ripper_translation.md +22 -0
  12. data/docs/serialization.md +3 -0
  13. data/ext/prism/api_node.c +2863 -2079
  14. data/ext/prism/extconf.rb +14 -37
  15. data/ext/prism/extension.c +241 -391
  16. data/ext/prism/extension.h +2 -2
  17. data/include/prism/ast.h +2156 -453
  18. data/include/prism/defines.h +58 -7
  19. data/include/prism/diagnostic.h +24 -6
  20. data/include/prism/node.h +0 -21
  21. data/include/prism/options.h +94 -3
  22. data/include/prism/parser.h +82 -40
  23. data/include/prism/regexp.h +18 -8
  24. data/include/prism/static_literals.h +3 -2
  25. data/include/prism/util/pm_char.h +1 -2
  26. data/include/prism/util/pm_constant_pool.h +0 -8
  27. data/include/prism/util/pm_integer.h +22 -15
  28. data/include/prism/util/pm_newline_list.h +11 -0
  29. data/include/prism/util/pm_string.h +28 -12
  30. data/include/prism/version.h +3 -3
  31. data/include/prism.h +47 -11
  32. data/lib/prism/compiler.rb +3 -0
  33. data/lib/prism/desugar_compiler.rb +111 -74
  34. data/lib/prism/dispatcher.rb +16 -1
  35. data/lib/prism/dot_visitor.rb +55 -34
  36. data/lib/prism/dsl.rb +660 -468
  37. data/lib/prism/ffi.rb +113 -8
  38. data/lib/prism/inspect_visitor.rb +296 -64
  39. data/lib/prism/lex_compat.rb +1 -1
  40. data/lib/prism/mutation_compiler.rb +11 -6
  41. data/lib/prism/node.rb +4262 -5023
  42. data/lib/prism/node_ext.rb +91 -14
  43. data/lib/prism/parse_result/comments.rb +0 -7
  44. data/lib/prism/parse_result/errors.rb +65 -0
  45. data/lib/prism/parse_result/newlines.rb +101 -11
  46. data/lib/prism/parse_result.rb +183 -6
  47. data/lib/prism/reflection.rb +12 -10
  48. data/lib/prism/relocation.rb +504 -0
  49. data/lib/prism/serialize.rb +496 -609
  50. data/lib/prism/string_query.rb +30 -0
  51. data/lib/prism/translation/parser/compiler.rb +185 -155
  52. data/lib/prism/translation/parser/lexer.rb +26 -4
  53. data/lib/prism/translation/parser.rb +9 -4
  54. data/lib/prism/translation/ripper.rb +23 -25
  55. data/lib/prism/translation/ruby_parser.rb +86 -17
  56. data/lib/prism/visitor.rb +3 -0
  57. data/lib/prism.rb +6 -8
  58. data/prism.gemspec +9 -5
  59. data/rbi/prism/dsl.rbi +521 -0
  60. data/rbi/prism/node.rbi +1115 -1120
  61. data/rbi/prism/parse_result.rbi +29 -0
  62. data/rbi/prism/string_query.rbi +12 -0
  63. data/rbi/prism/visitor.rbi +3 -0
  64. data/rbi/prism.rbi +36 -30
  65. data/sig/prism/dsl.rbs +190 -303
  66. data/sig/prism/mutation_compiler.rbs +1 -0
  67. data/sig/prism/node.rbs +678 -632
  68. data/sig/prism/parse_result.rbs +22 -0
  69. data/sig/prism/relocation.rbs +185 -0
  70. data/sig/prism/string_query.rbs +11 -0
  71. data/sig/prism/visitor.rbs +1 -0
  72. data/sig/prism.rbs +103 -64
  73. data/src/diagnostic.c +64 -28
  74. data/src/node.c +502 -1739
  75. data/src/options.c +76 -27
  76. data/src/prettyprint.c +188 -112
  77. data/src/prism.c +3376 -2293
  78. data/src/regexp.c +208 -71
  79. data/src/serialize.c +182 -50
  80. data/src/static_literals.c +64 -85
  81. data/src/token_type.c +4 -4
  82. data/src/util/pm_char.c +1 -1
  83. data/src/util/pm_constant_pool.c +0 -8
  84. data/src/util/pm_integer.c +53 -25
  85. data/src/util/pm_newline_list.c +29 -0
  86. data/src/util/pm_string.c +131 -80
  87. data/src/util/pm_strpbrk.c +32 -6
  88. metadata +11 -7
  89. data/include/prism/util/pm_string_list.h +0 -44
  90. data/lib/prism/debug.rb +0 -249
  91. data/lib/prism/translation/parser/rubocop.rb +0 -73
  92. data/src/util/pm_string_list.c +0 -28
data/lib/prism/debug.rb DELETED
@@ -1,249 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Prism
4
- # This module is used for testing and debugging and is not meant to be used by
5
- # consumers of this library.
6
- module Debug
7
- # A wrapper around a RubyVM::InstructionSequence that provides a more
8
- # convenient interface for accessing parts of the iseq.
9
- class ISeq # :nodoc:
10
- attr_reader :parts
11
-
12
- def initialize(parts)
13
- @parts = parts
14
- end
15
-
16
- def type
17
- parts[0]
18
- end
19
-
20
- def local_table
21
- parts[10]
22
- end
23
-
24
- def instructions
25
- parts[13]
26
- end
27
-
28
- def each_child
29
- instructions.each do |instruction|
30
- # Only look at arrays. Other instructions are line numbers or
31
- # tracepoint events.
32
- next unless instruction.is_a?(Array)
33
-
34
- instruction.each do |opnd|
35
- # Only look at arrays. Other operands are literals.
36
- next unless opnd.is_a?(Array)
37
-
38
- # Only look at instruction sequences. Other operands are literals.
39
- next unless opnd[0] == "YARVInstructionSequence/SimpleDataFormat"
40
-
41
- yield ISeq.new(opnd)
42
- end
43
- end
44
- end
45
- end
46
-
47
- private_constant :ISeq
48
-
49
- # :call-seq:
50
- # Debug::cruby_locals(source) -> Array
51
- #
52
- # For the given source, compiles with CRuby and returns a list of all of the
53
- # sets of local variables that were encountered.
54
- def self.cruby_locals(source)
55
- verbose, $VERBOSE = $VERBOSE, nil
56
-
57
- begin
58
- locals = [] #: Array[Array[Symbol | Integer]]
59
- stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)]
60
-
61
- while (iseq = stack.pop)
62
- names = [*iseq.local_table]
63
- names.map!.with_index do |name, index|
64
- # When an anonymous local variable is present in the iseq's local
65
- # table, it is represented as the stack offset from the top.
66
- # However, when these are dumped to binary and read back in, they
67
- # are replaced with the symbol :#arg_rest. To consistently handle
68
- # this, we replace them here with their index.
69
- if name == :"#arg_rest"
70
- names.length - index + 1
71
- else
72
- name
73
- end
74
- end
75
-
76
- locals << names
77
- iseq.each_child { |child| stack << child }
78
- end
79
-
80
- locals
81
- ensure
82
- $VERBOSE = verbose
83
- end
84
- end
85
-
86
- # Used to hold the place of a local that will be in the local table but
87
- # cannot be accessed directly from the source code. For example, the
88
- # iteration variable in a for loop or the positional parameter on a method
89
- # definition that is destructured.
90
- AnonymousLocal = Object.new
91
- private_constant :AnonymousLocal
92
-
93
- # :call-seq:
94
- # Debug::prism_locals(source) -> Array
95
- #
96
- # For the given source, parses with prism and returns a list of all of the
97
- # sets of local variables that were encountered.
98
- def self.prism_locals(source)
99
- locals = [] #: Array[Array[Symbol | Integer]]
100
- stack = [Prism.parse(source).value] #: Array[Prism::node]
101
-
102
- while (node = stack.pop)
103
- case node
104
- when BlockNode, DefNode, LambdaNode
105
- names = node.locals
106
- params =
107
- if node.is_a?(DefNode)
108
- node.parameters
109
- elsif node.parameters.is_a?(NumberedParametersNode)
110
- nil
111
- else
112
- node.parameters&.parameters
113
- end
114
-
115
- # prism places parameters in the same order that they appear in the
116
- # source. CRuby places them in the order that they need to appear
117
- # according to their own internal calling convention. We mimic that
118
- # order here so that we can compare properly.
119
- if params
120
- sorted = [
121
- *params.requireds.map do |required|
122
- if required.is_a?(RequiredParameterNode)
123
- required.name
124
- else
125
- AnonymousLocal
126
- end
127
- end,
128
- *params.optionals.map(&:name),
129
- *((params.rest.name || :*) if params.rest && !params.rest.is_a?(ImplicitRestNode)),
130
- *params.posts.map do |post|
131
- if post.is_a?(RequiredParameterNode)
132
- post.name
133
- else
134
- AnonymousLocal
135
- end
136
- end,
137
- *params.keywords.grep(RequiredKeywordParameterNode).map(&:name),
138
- *params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
139
- ]
140
-
141
- sorted << AnonymousLocal if params.keywords.any?
142
-
143
- if params.keyword_rest.is_a?(ForwardingParameterNode)
144
- sorted.push(:*, :**, :&, :"...")
145
- elsif params.keyword_rest.is_a?(KeywordRestParameterNode)
146
- sorted << (params.keyword_rest.name || :**)
147
- end
148
-
149
- # Recurse down the parameter tree to find any destructured
150
- # parameters and add them after the other parameters.
151
- param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse
152
- while (param = param_stack.pop)
153
- case param
154
- when MultiTargetNode
155
- param_stack.concat(param.rights.reverse)
156
- param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name)
157
- param_stack.concat(param.lefts.reverse)
158
- when RequiredParameterNode
159
- sorted << param.name
160
- when SplatNode
161
- sorted << param.expression.name
162
- end
163
- end
164
-
165
- if params.block
166
- sorted << (params.block.name || :&)
167
- end
168
-
169
- names = sorted.concat(names - sorted)
170
- end
171
-
172
- names.map!.with_index do |name, index|
173
- if name == AnonymousLocal
174
- names.length - index + 1
175
- else
176
- name
177
- end
178
- end
179
-
180
- locals << names
181
- when ClassNode, ModuleNode, ProgramNode, SingletonClassNode
182
- locals << node.locals
183
- when ForNode
184
- locals << [2]
185
- when PostExecutionNode
186
- locals.push([], [])
187
- when InterpolatedRegularExpressionNode
188
- locals << [] if node.once?
189
- end
190
-
191
- stack.concat(node.compact_child_nodes)
192
- end
193
-
194
- locals
195
- end
196
-
197
- # :call-seq:
198
- # Debug::newlines(source) -> Array
199
- #
200
- # For the given source string, return the byte offsets of every newline in
201
- # the source.
202
- def self.newlines(source)
203
- Prism.parse(source).source.offsets
204
- end
205
-
206
- # A wrapping around prism's internal encoding data structures. This is used
207
- # for reflection and debugging purposes.
208
- class Encoding
209
- # The name of the encoding, that can be passed to Encoding.find.
210
- attr_reader :name
211
-
212
- # Initialize a new encoding with the given name and whether or not it is
213
- # a multibyte encoding.
214
- def initialize(name, multibyte)
215
- @name = name
216
- @multibyte = multibyte
217
- end
218
-
219
- # Whether or not the encoding is a multibyte encoding.
220
- def multibyte?
221
- @multibyte
222
- end
223
-
224
- # Returns the number of bytes of the first character in the source string,
225
- # if it is valid for the encoding. Otherwise, returns 0.
226
- def width(source)
227
- Encoding._width(name, source)
228
- end
229
-
230
- # Returns true if the first character in the source string is a valid
231
- # alphanumeric character for the encoding.
232
- def alnum?(source)
233
- Encoding._alnum?(name, source)
234
- end
235
-
236
- # Returns true if the first character in the source string is a valid
237
- # alphabetic character for the encoding.
238
- def alpha?(source)
239
- Encoding._alpha?(name, source)
240
- end
241
-
242
- # Returns true if the first character in the source string is a valid
243
- # uppercase character for the encoding.
244
- def upper?(source)
245
- Encoding._upper?(name, source)
246
- end
247
- end
248
- end
249
- end
@@ -1,73 +0,0 @@
1
- # frozen_string_literal: true
2
- # typed: ignore
3
-
4
- warn "WARN: Prism is directly supported since RuboCop 1.62. The `prism/translation/parser/rubocop` file is deprecated."
5
-
6
- require "parser"
7
- require "rubocop"
8
-
9
- require_relative "../../prism"
10
- require_relative "../parser"
11
-
12
- module Prism
13
- module Translation
14
- class Parser
15
- # This is the special version numbers that should be used in RuboCop
16
- # configuration files to trigger using prism.
17
-
18
- # For Ruby 3.3
19
- VERSION_3_3 = 80_82_73_83_77.33
20
-
21
- # For Ruby 3.4
22
- VERSION_3_4 = 80_82_73_83_77.34
23
-
24
- # This module gets prepended into RuboCop::AST::ProcessedSource.
25
- module ProcessedSource
26
- # This condition is compatible with rubocop-ast versions up to 1.30.0.
27
- if RuboCop::AST::ProcessedSource.instance_method(:parser_class).arity == 1
28
- # Redefine parser_class so that we can inject the prism parser into the
29
- # list of known parsers.
30
- def parser_class(ruby_version)
31
- if ruby_version == Prism::Translation::Parser::VERSION_3_3
32
- warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \
33
- "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead."
34
- require_relative "../parser33"
35
- Prism::Translation::Parser33
36
- elsif ruby_version == Prism::Translation::Parser::VERSION_3_4
37
- warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \
38
- "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead."
39
- require_relative "../parser34"
40
- Prism::Translation::Parser34
41
- else
42
- super
43
- end
44
- end
45
- else
46
- # Redefine parser_class so that we can inject the prism parser into the
47
- # list of known parsers.
48
- def parser_class(ruby_version, _parser_engine)
49
- if ruby_version == Prism::Translation::Parser::VERSION_3_3
50
- warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \
51
- "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead."
52
- require_relative "../parser33"
53
- Prism::Translation::Parser33
54
- elsif ruby_version == Prism::Translation::Parser::VERSION_3_4
55
- warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \
56
- "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead."
57
- require_relative "../parser34"
58
- Prism::Translation::Parser34
59
- else
60
- super
61
- end
62
- end
63
- end
64
- end
65
- end
66
- end
67
- end
68
-
69
- # :stopdoc:
70
- RuboCop::AST::ProcessedSource.prepend(Prism::Translation::Parser::ProcessedSource)
71
- known_rubies = RuboCop::TargetRuby.const_get(:KNOWN_RUBIES)
72
- RuboCop::TargetRuby.send(:remove_const, :KNOWN_RUBIES)
73
- RuboCop::TargetRuby::KNOWN_RUBIES = [*known_rubies, Prism::Translation::Parser::VERSION_3_3].freeze
@@ -1,28 +0,0 @@
1
- #include "prism/util/pm_string_list.h"
2
-
3
- /**
4
- * Append a pm_string_t to the given string list.
5
- */
6
- void
7
- pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string) {
8
- if (string_list->length + 1 > string_list->capacity) {
9
- if (string_list->capacity == 0) {
10
- string_list->capacity = 1;
11
- } else {
12
- string_list->capacity *= 2;
13
- }
14
-
15
- string_list->strings = xrealloc(string_list->strings, string_list->capacity * sizeof(pm_string_t));
16
- if (string_list->strings == NULL) abort();
17
- }
18
-
19
- string_list->strings[string_list->length++] = *string;
20
- }
21
-
22
- /**
23
- * Free the memory associated with the string list
24
- */
25
- void
26
- pm_string_list_free(pm_string_list_t *string_list) {
27
- xfree(string_list->strings);
28
- }