prism 1.9.0 → 2.0.0.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +41 -12
  6. data/README.md +35 -97
  7. data/config.yml +174 -161
  8. data/docs/build_system.md +2 -2
  9. data/docs/configuration.md +5 -4
  10. data/docs/encoding.md +2 -2
  11. data/docs/fuzzing.md +4 -11
  12. data/docs/javascript.md +19 -4
  13. data/docs/mapping.md +1 -1
  14. data/docs/releasing.md +77 -5
  15. data/docs/relocation.md +2 -4
  16. data/docs/serialization.md +22 -24
  17. data/docs/testing.md +46 -29
  18. data/ext/prism/api_node.c +1461 -1401
  19. data/ext/prism/extconf.rb +3 -4
  20. data/ext/prism/extension.c +796 -422
  21. data/ext/prism/extension.h +3 -3
  22. data/include/prism/arena.h +37 -0
  23. data/include/prism/ast.h +2727 -610
  24. data/include/prism/buffer.h +52 -0
  25. data/include/prism/comments.h +55 -0
  26. data/include/prism/compiler/accel.h +26 -0
  27. data/include/prism/compiler/align.h +36 -0
  28. data/include/prism/compiler/assume.h +25 -0
  29. data/include/prism/compiler/bswap.h +46 -0
  30. data/include/prism/compiler/endianness.h +34 -0
  31. data/include/prism/compiler/exported.h +24 -0
  32. data/include/prism/compiler/fallthrough.h +22 -0
  33. data/include/prism/compiler/filesystem.h +32 -0
  34. data/include/prism/compiler/flex_array.h +19 -0
  35. data/include/prism/compiler/force_inline.h +21 -0
  36. data/include/prism/compiler/format.h +25 -0
  37. data/include/prism/compiler/inline.h +17 -0
  38. data/include/prism/compiler/nodiscard.h +22 -0
  39. data/include/prism/compiler/nonnull.h +18 -0
  40. data/include/prism/compiler/unused.h +18 -0
  41. data/include/prism/constant_pool.h +81 -0
  42. data/include/prism/diagnostic.h +38 -403
  43. data/include/prism/errors_format.h +53 -0
  44. data/include/prism/excludes.h +29 -0
  45. data/include/prism/integer.h +41 -0
  46. data/include/prism/internal/allocator.h +68 -0
  47. data/include/prism/internal/allocator_debug.h +88 -0
  48. data/include/prism/internal/arena.h +108 -0
  49. data/include/prism/internal/bit.h +42 -0
  50. data/include/prism/internal/buffer.h +94 -0
  51. data/include/prism/internal/char.h +145 -0
  52. data/include/prism/internal/comments.h +53 -0
  53. data/include/prism/internal/constant_pool.h +152 -0
  54. data/include/prism/internal/diagnostic.h +389 -0
  55. data/include/prism/{encoding.h → internal/encoding.h} +27 -68
  56. data/include/prism/internal/integer.h +68 -0
  57. data/include/prism/internal/isinf.h +16 -0
  58. data/include/prism/internal/line_offset_list.h +34 -0
  59. data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  60. data/include/prism/internal/magic_comments.h +23 -0
  61. data/include/prism/internal/memchr.h +15 -0
  62. data/include/prism/internal/node.h +38 -0
  63. data/include/prism/internal/options.h +232 -0
  64. data/include/prism/internal/parser.h +1004 -0
  65. data/include/prism/internal/regexp.h +26 -0
  66. data/include/prism/internal/serialize.h +39 -0
  67. data/include/prism/internal/source.h +72 -0
  68. data/include/prism/{static_literals.h → internal/static_literals.h} +26 -47
  69. data/include/prism/internal/stringy.h +35 -0
  70. data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  71. data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  72. data/include/prism/internal/tokens.h +11 -0
  73. data/include/prism/internal/xxh3.h +16 -0
  74. data/include/prism/json.h +32 -0
  75. data/include/prism/line_offset_list.h +61 -0
  76. data/include/prism/magic_comments.h +35 -0
  77. data/include/prism/node.h +18 -53
  78. data/include/prism/options.h +146 -280
  79. data/include/prism/parser.h +356 -867
  80. data/include/prism/prettyprint.h +6 -9
  81. data/include/prism/serialize.h +115 -0
  82. data/include/prism/source.h +155 -0
  83. data/include/prism/stream.h +28 -0
  84. data/include/prism/string_query.h +74 -0
  85. data/include/prism/stringy.h +72 -0
  86. data/include/prism/version.h +12 -3
  87. data/include/prism.h +43 -320
  88. data/lib/prism/compiler.rb +319 -305
  89. data/lib/prism/desugar_compiler.rb +105 -34
  90. data/lib/prism/dispatcher.rb +646 -473
  91. data/lib/prism/dot_visitor.rb +592 -511
  92. data/lib/prism/dsl.rb +391 -25
  93. data/lib/prism/ffi.rb +258 -90
  94. data/lib/prism/inspect_visitor.rb +357 -325
  95. data/lib/prism/lex_compat.rb +254 -212
  96. data/lib/prism/mutation_compiler.rb +313 -305
  97. data/lib/prism/node.rb +20966 -6992
  98. data/lib/prism/node_ext.rb +180 -187
  99. data/lib/prism/node_find.rb +215 -0
  100. data/lib/prism/parse_result/errors.rb +7 -1
  101. data/lib/prism/parse_result/newlines.rb +276 -8
  102. data/lib/prism/parse_result.rb +468 -139
  103. data/lib/prism/pattern.rb +65 -20
  104. data/lib/prism/reflection.rb +23 -12
  105. data/lib/prism/relocation.rb +149 -87
  106. data/lib/prism/serialize.rb +3460 -539
  107. data/lib/prism/string_query.rb +24 -1
  108. data/lib/prism/translation/parser/builder.rb +10 -2
  109. data/lib/prism/translation/parser/compiler.rb +44 -52
  110. data/lib/prism/translation/parser/lexer.rb +41 -92
  111. data/lib/prism/translation/parser.rb +1 -1
  112. data/lib/prism/translation/parser_current.rb +1 -1
  113. data/lib/prism/translation/ripper/lexer.rb +8 -10
  114. data/lib/prism/translation/ripper/sexp.rb +2 -10
  115. data/lib/prism/translation/ripper/shim.rb +2 -0
  116. data/lib/prism/translation/ripper.rb +967 -209
  117. data/lib/prism/translation/ruby_parser.rb +73 -316
  118. data/lib/prism/translation.rb +2 -0
  119. data/lib/prism/visitor.rb +632 -5
  120. data/lib/prism.rb +79 -9
  121. data/prism.gemspec +128 -62
  122. data/rbi/generated/prism/compiler.rbi +489 -0
  123. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  124. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  125. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  126. data/rbi/generated/prism/dsl.rbi +751 -0
  127. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  128. data/rbi/generated/prism/lex_compat.rbi +162 -0
  129. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  130. data/rbi/generated/prism/node.rbi +14407 -0
  131. data/rbi/generated/prism/node_ext.rbi +259 -0
  132. data/rbi/generated/prism/node_find.rbi +79 -0
  133. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  134. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  135. data/rbi/generated/prism/parse_result.rbi +776 -0
  136. data/rbi/generated/prism/pattern.rbi +134 -0
  137. data/rbi/generated/prism/reflection.rbi +89 -0
  138. data/rbi/generated/prism/relocation.rbi +329 -0
  139. data/rbi/generated/prism/serialize.rbi +184 -0
  140. data/rbi/generated/prism/string_query.rbi +43 -0
  141. data/rbi/generated/prism/translation.rbi +7 -0
  142. data/rbi/generated/prism/visitor.rbi +650 -0
  143. data/rbi/generated/prism.rbi +86 -0
  144. data/rbi/prism/translation/parser_versions.rbi +5 -0
  145. data/rbi/rubyvm/node_find.rbi +14 -0
  146. data/sig/generated/prism/compiler.rbs +492 -0
  147. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  148. data/sig/generated/prism/dispatcher.rbs +1292 -0
  149. data/sig/generated/prism/dot_visitor.rbs +635 -0
  150. data/sig/generated/prism/dsl.rbs +928 -0
  151. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  152. data/sig/generated/prism/lex_compat.rbs +166 -0
  153. data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
  154. data/sig/generated/prism/node.rbs +20896 -0
  155. data/sig/generated/prism/node_ext.rbs +290 -0
  156. data/sig/generated/prism/node_find.rbs +90 -0
  157. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  158. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  159. data/sig/generated/prism/parse_result.rbs +871 -0
  160. data/sig/generated/prism/pattern.rbs +156 -0
  161. data/sig/generated/prism/reflection.rbs +101 -0
  162. data/sig/generated/prism/relocation.rbs +428 -0
  163. data/sig/generated/prism/serialize.rbs +198 -0
  164. data/sig/generated/prism/string_query.rbs +43 -0
  165. data/sig/generated/prism/translation.rbs +8 -0
  166. data/sig/generated/prism/visitor.rbs +1111 -0
  167. data/sig/generated/prism.rbs +84 -0
  168. data/sig/manifest.yaml +2 -0
  169. data/src/arena.c +117 -0
  170. data/src/{util/pm_buffer.c → buffer.c} +53 -21
  171. data/src/{util/pm_char.c → char.c} +28 -57
  172. data/src/comments.c +454 -0
  173. data/src/constant_pool.c +406 -0
  174. data/src/diagnostic.c +97 -65
  175. data/src/encoding.c +8 -3
  176. data/src/errors_format.c +519 -0
  177. data/src/{util/pm_integer.c → integer.c} +33 -22
  178. data/src/json.c +5733 -0
  179. data/src/line_offset_list.c +100 -0
  180. data/src/list.c +24 -0
  181. data/src/{util/pm_memchr.c → memchr.c} +8 -6
  182. data/src/node.c +3504 -7674
  183. data/src/options.c +162 -49
  184. data/src/parser.c +367 -0
  185. data/src/prettyprint.c +421 -354
  186. data/src/prism.c +8402 -7161
  187. data/src/regexp.c +961 -45
  188. data/src/serialize.c +460 -424
  189. data/src/source.c +518 -0
  190. data/src/static_literals.c +111 -43
  191. data/src/string_query.c +180 -0
  192. data/src/stringy.c +91 -0
  193. data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  194. data/src/strpbrk.c +447 -0
  195. data/src/{token_type.c → tokens.c} +39 -18
  196. data/src/xxh3.c +292 -0
  197. metadata +128 -63
  198. data/ext/prism/api_pack.c +0 -276
  199. data/include/prism/defines.h +0 -260
  200. data/include/prism/pack.h +0 -163
  201. data/include/prism/regexp.h +0 -43
  202. data/include/prism/util/pm_buffer.h +0 -236
  203. data/include/prism/util/pm_char.h +0 -204
  204. data/include/prism/util/pm_constant_pool.h +0 -218
  205. data/include/prism/util/pm_integer.h +0 -130
  206. data/include/prism/util/pm_memchr.h +0 -29
  207. data/include/prism/util/pm_newline_list.h +0 -113
  208. data/include/prism/util/pm_string.h +0 -200
  209. data/lib/prism/pack.rb +0 -230
  210. data/lib/prism/parse_result/comments.rb +0 -188
  211. data/rbi/prism/compiler.rbi +0 -12
  212. data/rbi/prism/dsl.rbi +0 -524
  213. data/rbi/prism/inspect_visitor.rbi +0 -12
  214. data/rbi/prism/node.rbi +0 -8750
  215. data/rbi/prism/node_ext.rbi +0 -107
  216. data/rbi/prism/parse_result.rbi +0 -404
  217. data/rbi/prism/reflection.rbi +0 -58
  218. data/rbi/prism/string_query.rbi +0 -12
  219. data/rbi/prism/visitor.rbi +0 -473
  220. data/rbi/prism.rbi +0 -63
  221. data/sig/prism/compiler.rbs +0 -9
  222. data/sig/prism/dispatcher.rbs +0 -19
  223. data/sig/prism/dot_visitor.rbs +0 -6
  224. data/sig/prism/dsl.rbs +0 -351
  225. data/sig/prism/inspect_visitor.rbs +0 -22
  226. data/sig/prism/lex_compat.rbs +0 -10
  227. data/sig/prism/node.rbs +0 -4042
  228. data/sig/prism/node_ext.rbs +0 -149
  229. data/sig/prism/pack.rbs +0 -43
  230. data/sig/prism/parse_result/comments.rbs +0 -38
  231. data/sig/prism/parse_result.rbs +0 -197
  232. data/sig/prism/pattern.rbs +0 -13
  233. data/sig/prism/reflection.rbs +0 -50
  234. data/sig/prism/relocation.rbs +0 -185
  235. data/sig/prism/serialize.rbs +0 -8
  236. data/sig/prism/string_query.rbs +0 -11
  237. data/sig/prism/visitor.rbs +0 -169
  238. data/sig/prism.rbs +0 -272
  239. data/src/pack.c +0 -509
  240. data/src/util/pm_constant_pool.c +0 -342
  241. data/src/util/pm_list.c +0 -49
  242. data/src/util/pm_newline_list.c +0 -125
  243. data/src/util/pm_string.c +0 -381
  244. data/src/util/pm_strpbrk.c +0 -206
@@ -0,0 +1,184 @@
1
+ # typed: true
2
+
3
+ module Prism
4
+ # A module responsible for deserializing parse results.
5
+ module Serialize
6
+ # The major version of prism that we are expecting to find in the serialized
7
+ # strings.
8
+ MAJOR_VERSION = T.let(nil, Integer)
9
+
10
+ # The minor version of prism that we are expecting to find in the serialized
11
+ # strings.
12
+ MINOR_VERSION = T.let(nil, Integer)
13
+
14
+ # The patch version of prism that we are expecting to find in the serialized
15
+ # strings.
16
+ PATCH_VERSION = T.let(nil, Integer)
17
+
18
+ # Deserialize the dumped output from a request to parse or parse_file.
19
+ #
20
+ # The formatting of the source of this method is purposeful to illustrate
21
+ # the structure of the serialized data.
22
+ sig { params(input: String, serialized: String, freeze: T::Boolean).returns(ParseResult) }
23
+ def self.load_parse(input, serialized, freeze); end
24
+
25
+ # Deserialize the dumped output from a request to lex or lex_file.
26
+ #
27
+ # The formatting of the source of this method is purposeful to illustrate
28
+ # the structure of the serialized data.
29
+ sig { params(input: String, serialized: String, freeze: T::Boolean).returns(LexResult) }
30
+ def self.load_lex(input, serialized, freeze); end
31
+
32
+ # Deserialize the dumped output from a request to parse_comments or
33
+ # parse_file_comments.
34
+ #
35
+ # The formatting of the source of this method is purposeful to illustrate
36
+ # the structure of the serialized data.
37
+ sig { params(input: String, serialized: String, freeze: T::Boolean).returns(T::Array[Comment]) }
38
+ def self.load_parse_comments(input, serialized, freeze); end
39
+
40
+ # Deserialize the dumped output from a request to parse_lex or
41
+ # parse_lex_file.
42
+ #
43
+ # The formatting of the source of this method is purposeful to illustrate
44
+ # the structure of the serialized data.
45
+ sig { params(input: String, serialized: String, freeze: T::Boolean).returns(ParseLexResult) }
46
+ def self.load_parse_lex(input, serialized, freeze); end
47
+
48
+ class ConstantPool
49
+ sig { returns(Integer) }
50
+ attr_reader :size
51
+
52
+ sig { params(serialized: String, base: Integer, size: Integer).void }
53
+ def initialize(serialized, base, size); end
54
+
55
+ sig { params(index: Integer, encoding: Encoding).returns(Symbol) }
56
+ def get(index, encoding); end
57
+ end
58
+
59
+ FastStringIO = T.let(nil, ::T.untyped)
60
+
61
+ class Loader
62
+ sig { returns(String) }
63
+ attr_reader :input
64
+
65
+ sig { returns(StringIO) }
66
+ attr_reader :io
67
+
68
+ sig { returns(Source) }
69
+ attr_reader :source
70
+
71
+ sig { params(source: Source, serialized: String).void }
72
+ def initialize(source, serialized); end
73
+
74
+ sig { returns(T::Boolean) }
75
+ def eof?; end
76
+
77
+ sig { params(constant_pool: ConstantPool).void }
78
+ def load_constant_pool(constant_pool); end
79
+
80
+ sig { void }
81
+ def load_header; end
82
+
83
+ sig { returns(Encoding) }
84
+ def load_encoding; end
85
+
86
+ sig { params(freeze: T::Boolean).returns(T::Array[Integer]) }
87
+ def load_line_offsets(freeze); end
88
+
89
+ # Load the comments attached to each node into the comment attachments and
90
+ # comment ranges of the given source. For each node that owns comments, the
91
+ # index at which its comments begin is stored at its node id and the index
92
+ # at which they end at the following node id.
93
+ sig { params(source: Source, freeze: T::Boolean).void }
94
+ def load_comment_attachments(source, freeze); end
95
+
96
+ # Load the comments into the given list, which belongs to the source that
97
+ # the comments refer to.
98
+ sig { params(comments: T::Array[Comment], freeze: T::Boolean).returns(T::Array[Comment]) }
99
+ def load_comments(comments, freeze); end
100
+
101
+ sig { params(freeze: T::Boolean).returns(T::Array[MagicComment]) }
102
+ def load_magic_comments(freeze); end
103
+
104
+ DIAGNOSTIC_TYPES = T.let(nil, T::Array[Symbol])
105
+
106
+ sig { returns(Symbol) }
107
+ def load_error_level; end
108
+
109
+ sig { params(encoding: Encoding, freeze: T::Boolean).returns(T::Array[ParseError]) }
110
+ def load_errors(encoding, freeze); end
111
+
112
+ sig { returns(Symbol) }
113
+ def load_warning_level; end
114
+
115
+ sig { params(encoding: Encoding, freeze: T::Boolean).returns(T::Array[ParseWarning]) }
116
+ def load_warnings(encoding, freeze); end
117
+
118
+ sig { returns(T::Array[Token]) }
119
+ def load_tokens; end
120
+
121
+ # variable-length integer using https://en.wikipedia.org/wiki/LEB128
122
+ # This is also what protobuf uses: https://protobuf.dev/programming-guides/encoding/#varints
123
+ sig { returns(Integer) }
124
+ def load_varuint; end
125
+
126
+ sig { returns(Integer) }
127
+ def load_varsint; end
128
+
129
+ sig { returns(Integer) }
130
+ def load_integer; end
131
+
132
+ sig { returns(Float) }
133
+ def load_double; end
134
+
135
+ sig { returns(T::Boolean) }
136
+ def load_bool; end
137
+
138
+ sig { returns(Integer) }
139
+ def load_uint32; end
140
+
141
+ sig { returns(Integer) }
142
+ def load_uint64; end
143
+
144
+ sig { params(constant_pool: ConstantPool, encoding: Encoding, freeze: T::Boolean).returns(::T.nilable(Node)) }
145
+ def load_optional_node(constant_pool, encoding, freeze); end
146
+
147
+ sig { params(encoding: Encoding).returns(String) }
148
+ def load_string(encoding); end
149
+
150
+ sig { params(freeze: T::Boolean).returns(Location) }
151
+ def load_location_object(freeze); end
152
+
153
+ # Load a location object from the serialized data. Note that we are lying
154
+ # about the signature a bit here, because we sometimes load it as a packed
155
+ # integer instead of an object.
156
+ sig { params(freeze: T::Boolean).returns(Location) }
157
+ def load_location(freeze); end
158
+
159
+ # Load an optional location object from the serialized data if it is
160
+ # present. Note that we are lying about the signature a bit here, because
161
+ # we sometimes load it as a packed integer instead of an object.
162
+ sig { params(freeze: T::Boolean).returns(::T.nilable(Location)) }
163
+ def load_optional_location(freeze); end
164
+
165
+ sig { params(freeze: T::Boolean).returns(::T.nilable(Location)) }
166
+ def load_optional_location_object(freeze); end
167
+
168
+ sig { params(constant_pool: ConstantPool, encoding: Encoding).returns(Symbol) }
169
+ def load_constant(constant_pool, encoding); end
170
+
171
+ sig { params(constant_pool: ConstantPool, encoding: Encoding).returns(::T.nilable(Symbol)) }
172
+ def load_optional_constant(constant_pool, encoding); end
173
+
174
+ sig { params(constant_pool: ConstantPool, encoding: Encoding, freeze: T::Boolean).returns(Node) }
175
+ def load_node(constant_pool, encoding, freeze); end
176
+
177
+ sig { void }
178
+ def define_load_node_lambdas; end
179
+ end
180
+
181
+ # The token types that can be indexed by their enum values.
182
+ TOKEN_TYPES = T.let(nil, T::Array[::T.nilable(Symbol)])
183
+ end
184
+ end
@@ -0,0 +1,43 @@
1
+ # typed: true
2
+
3
+ module Prism
4
+ # Query methods that allow categorizing strings based on their context for
5
+ # where they could be valid in a Ruby syntax tree.
6
+ class StringQuery
7
+ sig { params(string: String).returns(T::Boolean) }
8
+ def self.local?(string); end
9
+
10
+ sig { params(string: String).returns(T::Boolean) }
11
+ def self.constant?(string); end
12
+
13
+ sig { params(string: String).returns(T::Boolean) }
14
+ def self.method_name?(string); end
15
+
16
+ sig { params(string: String).returns(T::Boolean) }
17
+ def self.keyword?(string); end
18
+
19
+ # The string that this query is wrapping.
20
+ sig { returns(String) }
21
+ attr_reader :string
22
+
23
+ # Initialize a new query with the given string.
24
+ sig { params(string: String).void }
25
+ def initialize(string); end
26
+
27
+ # Whether or not this string is a valid local variable name.
28
+ sig { returns(T::Boolean) }
29
+ def local?; end
30
+
31
+ # Whether or not this string is a valid constant name.
32
+ sig { returns(T::Boolean) }
33
+ def constant?; end
34
+
35
+ # Whether or not this string is a valid method name.
36
+ sig { returns(T::Boolean) }
37
+ def method_name?; end
38
+
39
+ # Whether or not this string is a valid keyword.
40
+ sig { returns(T::Boolean) }
41
+ def keyword?; end
42
+ end
43
+ end
@@ -0,0 +1,7 @@
1
+ # typed: true
2
+
3
+ module Prism
4
+ # This module is responsible for converting the prism syntax tree into other
5
+ # syntax trees.
6
+ module Translation; end
7
+ end