prism 1.9.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +144 -40
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +49 -20
  6. data/README.md +38 -98
  7. data/config.yml +174 -161
  8. data/docs/build_system.md +2 -2
  9. data/docs/configuration.md +5 -4
  10. data/docs/encoding.md +2 -2
  11. data/docs/fuzzing.md +9 -16
  12. data/docs/javascript.md +19 -4
  13. data/docs/mapping.md +1 -1
  14. data/docs/releasing.md +77 -5
  15. data/docs/relocation.md +2 -4
  16. data/docs/serialization.md +39 -28
  17. data/docs/testing.md +46 -29
  18. data/ext/prism/api_node.c +1527 -1467
  19. data/ext/prism/extconf.rb +3 -4
  20. data/ext/prism/extension.c +759 -418
  21. data/ext/prism/extension.h +3 -3
  22. data/include/prism/arena.h +45 -0
  23. data/include/prism/arena.hpp +42 -0
  24. data/include/prism/ast.h +2738 -611
  25. data/include/prism/ast.hpp +641 -0
  26. data/include/prism/buffer.h +60 -0
  27. data/include/prism/buffer.hpp +55 -0
  28. data/include/prism/comments.h +72 -0
  29. data/include/prism/compiler/accel.h +26 -0
  30. data/include/prism/compiler/align.h +36 -0
  31. data/include/prism/compiler/assume.h +25 -0
  32. data/include/prism/compiler/branch.h +26 -0
  33. data/include/prism/compiler/bswap.h +46 -0
  34. data/include/prism/compiler/cplusplus.hpp +16 -0
  35. data/include/prism/compiler/endianness.h +34 -0
  36. data/include/prism/compiler/exceptions.hpp +17 -0
  37. data/include/prism/compiler/exported.h +24 -0
  38. data/include/prism/compiler/fallthrough.h +22 -0
  39. data/include/prism/compiler/filesystem.h +32 -0
  40. data/include/prism/compiler/flex_array.h +19 -0
  41. data/include/prism/compiler/force_inline.h +21 -0
  42. data/include/prism/compiler/format.h +27 -0
  43. data/include/prism/compiler/inline.h +17 -0
  44. data/include/prism/compiler/lifetimebound.hpp +25 -0
  45. data/include/prism/compiler/nodiscard.h +22 -0
  46. data/include/prism/compiler/nonnull.h +18 -0
  47. data/include/prism/compiler/unused.h +18 -0
  48. data/include/prism/constant_pool.h +106 -0
  49. data/include/prism/diagnostic.h +55 -402
  50. data/include/prism/errors_format.h +61 -0
  51. data/include/prism/excludes.h +29 -0
  52. data/include/prism/integer.h +41 -0
  53. data/include/prism/internal/allocator.h +68 -0
  54. data/include/prism/internal/allocator_debug.h +88 -0
  55. data/include/prism/internal/arena.h +108 -0
  56. data/include/prism/internal/bit.h +42 -0
  57. data/include/prism/internal/buffer.h +94 -0
  58. data/include/prism/internal/char.h +145 -0
  59. data/include/prism/internal/comments.h +54 -0
  60. data/include/prism/internal/constant_pool.h +152 -0
  61. data/include/prism/internal/diagnostic.h +389 -0
  62. data/include/prism/{encoding.h → internal/encoding.h} +27 -68
  63. data/include/prism/internal/integer.h +68 -0
  64. data/include/prism/internal/isinf.h +16 -0
  65. data/include/prism/internal/line_offset_list.h +34 -0
  66. data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  67. data/include/prism/internal/magic_comments.h +23 -0
  68. data/include/prism/internal/memchr.h +15 -0
  69. data/include/prism/internal/node.h +42 -0
  70. data/include/prism/internal/options.h +232 -0
  71. data/include/prism/internal/parser.h +1015 -0
  72. data/include/prism/internal/regexp.h +26 -0
  73. data/include/prism/internal/serialize.h +39 -0
  74. data/include/prism/internal/source.h +72 -0
  75. data/include/prism/internal/static_literals.h +60 -0
  76. data/include/prism/internal/stringy.h +30 -0
  77. data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  78. data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  79. data/include/prism/internal/tokens.h +11 -0
  80. data/include/prism/internal/xxh3.h +16 -0
  81. data/include/prism/json.h +40 -0
  82. data/include/prism/line_offset_list.h +69 -0
  83. data/include/prism/magic_comments.h +53 -0
  84. data/include/prism/node.h +59 -49
  85. data/include/prism/node.hpp +272 -0
  86. data/include/prism/node_type.hpp +38 -0
  87. data/include/prism/options.h +154 -279
  88. data/include/prism/options.hpp +43 -0
  89. data/include/prism/parser.h +367 -870
  90. data/include/prism/parser.hpp +274 -0
  91. data/include/prism/prettyprint.h +14 -9
  92. data/include/prism/serialize.h +123 -0
  93. data/include/prism/source.h +163 -0
  94. data/include/prism/stream.h +36 -0
  95. data/include/prism/string_query.h +82 -0
  96. data/include/prism/stringy.h +61 -0
  97. data/include/prism/stringy.hpp +31 -0
  98. data/include/prism/version.h +20 -3
  99. data/include/prism.h +35 -320
  100. data/include/prism.hpp +24 -0
  101. data/lib/prism/compiler.rb +319 -305
  102. data/lib/prism/desugar_compiler.rb +105 -34
  103. data/lib/prism/dispatcher.rb +646 -473
  104. data/lib/prism/dot_visitor.rb +592 -511
  105. data/lib/prism/dsl.rb +391 -25
  106. data/lib/prism/ffi.rb +258 -90
  107. data/lib/prism/inspect_visitor.rb +357 -325
  108. data/lib/prism/lex_compat.rb +254 -212
  109. data/lib/prism/mutation_compiler.rb +313 -305
  110. data/lib/prism/node.rb +20966 -6992
  111. data/lib/prism/node_ext.rb +180 -187
  112. data/lib/prism/node_find.rb +215 -0
  113. data/lib/prism/parse_result/errors.rb +7 -1
  114. data/lib/prism/parse_result/newlines.rb +276 -8
  115. data/lib/prism/parse_result.rb +468 -139
  116. data/lib/prism/pattern.rb +65 -20
  117. data/lib/prism/reflection.rb +23 -12
  118. data/lib/prism/relocation.rb +149 -87
  119. data/lib/prism/serialize.rb +2889 -700
  120. data/lib/prism/string_query.rb +24 -1
  121. data/lib/prism/translation/parser/builder.rb +10 -2
  122. data/lib/prism/translation/parser/compiler.rb +44 -52
  123. data/lib/prism/translation/parser/lexer.rb +41 -92
  124. data/lib/prism/translation/parser.rb +1 -1
  125. data/lib/prism/translation/parser_current.rb +1 -1
  126. data/lib/prism/translation/ripper/lexer.rb +8 -10
  127. data/lib/prism/translation/ripper/sexp.rb +2 -10
  128. data/lib/prism/translation/ripper/shim.rb +2 -0
  129. data/lib/prism/translation/ripper.rb +967 -209
  130. data/lib/prism/translation/ruby_parser.rb +73 -316
  131. data/lib/prism/translation.rb +2 -0
  132. data/lib/prism/visitor.rb +632 -5
  133. data/lib/prism.rb +79 -9
  134. data/prism.gemspec +141 -62
  135. data/rbi/generated/prism/compiler.rbi +489 -0
  136. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  137. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  138. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  139. data/rbi/generated/prism/dsl.rbi +751 -0
  140. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  141. data/rbi/generated/prism/lex_compat.rbi +162 -0
  142. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  143. data/rbi/generated/prism/node.rbi +15143 -0
  144. data/rbi/generated/prism/node_ext.rbi +259 -0
  145. data/rbi/generated/prism/node_find.rbi +79 -0
  146. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  147. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  148. data/rbi/generated/prism/parse_result.rbi +776 -0
  149. data/rbi/generated/prism/pattern.rbi +134 -0
  150. data/rbi/generated/prism/reflection.rbi +89 -0
  151. data/rbi/generated/prism/relocation.rbi +329 -0
  152. data/rbi/generated/prism/serialize.rbi +185 -0
  153. data/rbi/generated/prism/string_query.rbi +43 -0
  154. data/rbi/generated/prism/translation.rbi +7 -0
  155. data/rbi/generated/prism/visitor.rbi +650 -0
  156. data/rbi/generated/prism.rbi +86 -0
  157. data/rbi/prism/translation/parser_versions.rbi +5 -0
  158. data/rbi/rubyvm/node_find.rbi +14 -0
  159. data/sig/generated/prism/compiler.rbs +492 -0
  160. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  161. data/sig/generated/prism/dispatcher.rbs +1292 -0
  162. data/sig/generated/prism/dot_visitor.rbs +635 -0
  163. data/sig/generated/prism/dsl.rbs +928 -0
  164. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  165. data/sig/generated/prism/lex_compat.rbs +166 -0
  166. data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
  167. data/sig/generated/prism/node.rbs +21632 -0
  168. data/sig/generated/prism/node_ext.rbs +290 -0
  169. data/sig/generated/prism/node_find.rbs +90 -0
  170. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  171. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  172. data/sig/generated/prism/parse_result.rbs +871 -0
  173. data/sig/generated/prism/pattern.rbs +156 -0
  174. data/sig/generated/prism/reflection.rbs +101 -0
  175. data/sig/generated/prism/relocation.rbs +428 -0
  176. data/sig/generated/prism/serialize.rbs +200 -0
  177. data/sig/generated/prism/string_query.rbs +43 -0
  178. data/sig/generated/prism/translation.rbs +8 -0
  179. data/sig/generated/prism/visitor.rbs +1111 -0
  180. data/sig/generated/prism.rbs +84 -0
  181. data/sig/manifest.yaml +2 -0
  182. data/src/arena.c +122 -0
  183. data/src/{util/pm_buffer.c → buffer.c} +53 -21
  184. data/src/{util/pm_char.c → char.c} +28 -57
  185. data/src/comments.c +455 -0
  186. data/src/constant_pool.c +406 -0
  187. data/src/diagnostic.c +120 -68
  188. data/src/encoding.c +8 -3
  189. data/src/errors_format.c +519 -0
  190. data/src/{util/pm_integer.c → integer.c} +38 -22
  191. data/src/json.c +5733 -0
  192. data/src/line_offset_list.c +100 -0
  193. data/src/list.c +24 -0
  194. data/src/{util/pm_memchr.c → memchr.c} +8 -6
  195. data/src/node.c +3465 -7787
  196. data/src/options.c +161 -54
  197. data/src/parser.c +371 -0
  198. data/src/prettyprint.c +421 -354
  199. data/src/prism.c +8503 -7280
  200. data/src/regexp.c +960 -45
  201. data/src/serialize.c +2434 -2048
  202. data/src/source.c +524 -0
  203. data/src/static_literals.c +303 -360
  204. data/src/string_query.c +180 -0
  205. data/src/stringy.c +61 -0
  206. data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  207. data/src/strpbrk.c +447 -0
  208. data/src/{token_type.c → tokens.c} +39 -18
  209. data/src/xxh3.c +292 -0
  210. metadata +141 -63
  211. data/ext/prism/api_pack.c +0 -276
  212. data/include/prism/defines.h +0 -260
  213. data/include/prism/pack.h +0 -163
  214. data/include/prism/regexp.h +0 -43
  215. data/include/prism/static_literals.h +0 -121
  216. data/include/prism/util/pm_buffer.h +0 -236
  217. data/include/prism/util/pm_char.h +0 -204
  218. data/include/prism/util/pm_constant_pool.h +0 -218
  219. data/include/prism/util/pm_integer.h +0 -130
  220. data/include/prism/util/pm_memchr.h +0 -29
  221. data/include/prism/util/pm_newline_list.h +0 -113
  222. data/include/prism/util/pm_string.h +0 -200
  223. data/lib/prism/pack.rb +0 -230
  224. data/lib/prism/parse_result/comments.rb +0 -188
  225. data/rbi/prism/compiler.rbi +0 -12
  226. data/rbi/prism/dsl.rbi +0 -524
  227. data/rbi/prism/inspect_visitor.rbi +0 -12
  228. data/rbi/prism/node.rbi +0 -8750
  229. data/rbi/prism/node_ext.rbi +0 -107
  230. data/rbi/prism/parse_result.rbi +0 -404
  231. data/rbi/prism/reflection.rbi +0 -58
  232. data/rbi/prism/string_query.rbi +0 -12
  233. data/rbi/prism/visitor.rbi +0 -473
  234. data/rbi/prism.rbi +0 -63
  235. data/sig/prism/compiler.rbs +0 -9
  236. data/sig/prism/dispatcher.rbs +0 -19
  237. data/sig/prism/dot_visitor.rbs +0 -6
  238. data/sig/prism/dsl.rbs +0 -351
  239. data/sig/prism/inspect_visitor.rbs +0 -22
  240. data/sig/prism/lex_compat.rbs +0 -10
  241. data/sig/prism/node.rbs +0 -4042
  242. data/sig/prism/node_ext.rbs +0 -149
  243. data/sig/prism/pack.rbs +0 -43
  244. data/sig/prism/parse_result/comments.rbs +0 -38
  245. data/sig/prism/parse_result.rbs +0 -197
  246. data/sig/prism/pattern.rbs +0 -13
  247. data/sig/prism/reflection.rbs +0 -50
  248. data/sig/prism/relocation.rbs +0 -185
  249. data/sig/prism/serialize.rbs +0 -8
  250. data/sig/prism/string_query.rbs +0 -11
  251. data/sig/prism/visitor.rbs +0 -169
  252. data/sig/prism.rbs +0 -272
  253. data/src/pack.c +0 -509
  254. data/src/util/pm_constant_pool.c +0 -342
  255. data/src/util/pm_list.c +0 -49
  256. data/src/util/pm_newline_list.c +0 -125
  257. data/src/util/pm_string.c +0 -381
  258. data/src/util/pm_strpbrk.c +0 -206
@@ -0,0 +1,185 @@
1
+ # typed: true
2
+
3
+ module Prism
4
+ # A module responsible for deserializing parse results.
5
+ module Serialize
6
+ # The major version of prism that we are expecting to find in the serialized
7
+ # strings.
8
+ MAJOR_VERSION = T.let(nil, Integer)
9
+
10
+ # The minor version of prism that we are expecting to find in the serialized
11
+ # strings.
12
+ MINOR_VERSION = T.let(nil, Integer)
13
+
14
+ # The patch version of prism that we are expecting to find in the serialized
15
+ # strings.
16
+ PATCH_VERSION = T.let(nil, Integer)
17
+
18
+ # Deserialize the dumped output from a request to parse or parse_file.
19
+ #
20
+ # The formatting of the source of this method is purposeful to illustrate
21
+ # the structure of the serialized data.
22
+ sig { params(input: String, serialized: String, freeze: T::Boolean).returns(ParseResult) }
23
+ def self.load_parse(input, serialized, freeze); end
24
+
25
+ # Deserialize the dumped output from a request to lex or lex_file.
26
+ #
27
+ # The formatting of the source of this method is purposeful to illustrate
28
+ # the structure of the serialized data.
29
+ sig { params(input: String, serialized: String, freeze: T::Boolean).returns(LexResult) }
30
+ def self.load_lex(input, serialized, freeze); end
31
+
32
+ # Deserialize the dumped output from a request to parse_comments or
33
+ # parse_file_comments.
34
+ #
35
+ # The formatting of the source of this method is purposeful to illustrate
36
+ # the structure of the serialized data.
37
+ sig { params(input: String, serialized: String, freeze: T::Boolean).returns(T::Array[Comment]) }
38
+ def self.load_parse_comments(input, serialized, freeze); end
39
+
40
+ # Deserialize the dumped output from a request to parse_lex or
41
+ # parse_lex_file.
42
+ #
43
+ # The formatting of the source of this method is purposeful to illustrate
44
+ # the structure of the serialized data.
45
+ sig { params(input: String, serialized: String, freeze: T::Boolean).returns(ParseLexResult) }
46
+ def self.load_parse_lex(input, serialized, freeze); end
47
+
48
+ class ConstantPool
49
+ sig { returns(Integer) }
50
+ attr_reader :size
51
+
52
+ sig { params(serialized: String, base: Integer, size: Integer).void }
53
+ def initialize(serialized, base, size); end
54
+
55
+ sig { params(index: Integer, encoding: Encoding).returns(Symbol) }
56
+ def get(index, encoding); end
57
+ end
58
+
59
+ FastStringIO = T.let(nil, ::T.untyped)
60
+
61
+ class Loader
62
+ sig { returns(String) }
63
+ attr_reader :input
64
+
65
+ sig { returns(StringIO) }
66
+ attr_reader :io
67
+
68
+ sig { returns(Source) }
69
+ attr_reader :source
70
+
71
+ sig { params(source: Source, serialized: String).void }
72
+ def initialize(source, serialized); end
73
+
74
+ sig { returns(T::Boolean) }
75
+ def eof?; end
76
+
77
+ sig { params(constant_pool: ConstantPool).void }
78
+ def load_constant_pool(constant_pool); end
79
+
80
+ sig { void }
81
+ def load_header; end
82
+
83
+ sig { returns(Encoding) }
84
+ def load_encoding; end
85
+
86
+ sig { params(freeze: T::Boolean).returns(T::Array[Integer]) }
87
+ def load_line_offsets(freeze); end
88
+
89
+ # Load the comments attached to each node into the comment attachments and
90
+ # comment ranges of the given source. For each node that owns comments, the
91
+ # index at which its comments begin is stored at its node id and the index
92
+ # at which they end at the following node id.
93
+ sig { params(source: Source, freeze: T::Boolean).void }
94
+ def load_comment_attachments(source, freeze); end
95
+
96
+ # Load the comments into the given list, which belongs to the source that
97
+ # the comments refer to.
98
+ sig { params(comments: T::Array[Comment], freeze: T::Boolean).returns(T::Array[Comment]) }
99
+ def load_comments(comments, freeze); end
100
+
101
+ sig { params(freeze: T::Boolean).returns(T::Array[MagicComment]) }
102
+ def load_magic_comments(freeze); end
103
+
104
+ DIAGNOSTIC_TYPES = T.let(nil, T::Array[Symbol])
105
+
106
+ sig { returns(Symbol) }
107
+ def load_error_level; end
108
+
109
+ sig { params(encoding: Encoding, freeze: T::Boolean).returns(T::Array[ParseError]) }
110
+ def load_errors(encoding, freeze); end
111
+
112
+ sig { returns(Symbol) }
113
+ def load_warning_level; end
114
+
115
+ sig { params(encoding: Encoding, freeze: T::Boolean).returns(T::Array[ParseWarning]) }
116
+ def load_warnings(encoding, freeze); end
117
+
118
+ sig { returns(T::Array[Token]) }
119
+ def load_tokens; end
120
+
121
+ # variable-length integer using https://en.wikipedia.org/wiki/LEB128
122
+ # This is also what protobuf uses: https://protobuf.dev/programming-guides/encoding/#varints
123
+ sig { returns(Integer) }
124
+ def load_varuint; end
125
+
126
+ sig { returns(Integer) }
127
+ def load_varsint; end
128
+
129
+ sig { returns(Integer) }
130
+ def load_integer; end
131
+
132
+ sig { returns(Float) }
133
+ def load_double; end
134
+
135
+ sig { returns(T::Boolean) }
136
+ def load_bool; end
137
+
138
+ sig { returns(Integer) }
139
+ def load_uint32; end
140
+
141
+ sig { returns(Integer) }
142
+ def load_uint64; end
143
+
144
+ sig { params(encoding: Encoding).returns(String) }
145
+ def load_string(encoding); end
146
+
147
+ sig { params(freeze: T::Boolean).returns(Location) }
148
+ def load_location_object(freeze); end
149
+
150
+ # Load a location object from the serialized data. Note that we are lying
151
+ # about the signature a bit here, because we sometimes load it as a packed
152
+ # integer instead of an object.
153
+ sig { params(freeze: T::Boolean).returns(Location) }
154
+ def load_location(freeze); end
155
+
156
+ # Load an optional location object from the serialized data if it is
157
+ # present. Note that we are lying about the signature a bit here, because
158
+ # we sometimes load it as a packed integer instead of an object.
159
+ sig { params(freeze: T::Boolean).returns(::T.nilable(Location)) }
160
+ def load_optional_location(freeze); end
161
+
162
+ sig { params(freeze: T::Boolean).returns(::T.nilable(Location)) }
163
+ def load_optional_location_object(freeze); end
164
+
165
+ sig { params(constant_pool: ConstantPool, encoding: Encoding).returns(Symbol) }
166
+ def load_constant(constant_pool, encoding); end
167
+
168
+ sig { params(constant_pool: ConstantPool, encoding: Encoding).returns(::T.nilable(Symbol)) }
169
+ def load_optional_constant(constant_pool, encoding); end
170
+
171
+ # Load the nodes that are serialized before the given offset and return
172
+ # the root. Nodes are serialized in post-order, so the children of each
173
+ # node are the topmost values on the stack, in field order, by the time
174
+ # its record is read.
175
+ sig { params(constant_pool: ConstantPool, encoding: Encoding, freeze: T::Boolean, finish: Integer).returns(Node) }
176
+ def load_nodes(constant_pool, encoding, freeze, finish); end
177
+
178
+ sig { void }
179
+ def define_load_node_lambdas; end
180
+ end
181
+
182
+ # The token types that can be indexed by their enum values.
183
+ TOKEN_TYPES = T.let(nil, T::Array[::T.nilable(Symbol)])
184
+ end
185
+ end
@@ -0,0 +1,43 @@
1
+ # typed: true
2
+
3
+ module Prism
4
+ # Query methods that allow categorizing strings based on their context for
5
+ # where they could be valid in a Ruby syntax tree.
6
+ class StringQuery
7
+ sig { params(string: String).returns(T::Boolean) }
8
+ def self.local?(string); end
9
+
10
+ sig { params(string: String).returns(T::Boolean) }
11
+ def self.constant?(string); end
12
+
13
+ sig { params(string: String).returns(T::Boolean) }
14
+ def self.method_name?(string); end
15
+
16
+ sig { params(string: String).returns(T::Boolean) }
17
+ def self.keyword?(string); end
18
+
19
+ # The string that this query is wrapping.
20
+ sig { returns(String) }
21
+ attr_reader :string
22
+
23
+ # Initialize a new query with the given string.
24
+ sig { params(string: String).void }
25
+ def initialize(string); end
26
+
27
+ # Whether or not this string is a valid local variable name.
28
+ sig { returns(T::Boolean) }
29
+ def local?; end
30
+
31
+ # Whether or not this string is a valid constant name.
32
+ sig { returns(T::Boolean) }
33
+ def constant?; end
34
+
35
+ # Whether or not this string is a valid method name.
36
+ sig { returns(T::Boolean) }
37
+ def method_name?; end
38
+
39
+ # Whether or not this string is a valid keyword.
40
+ sig { returns(T::Boolean) }
41
+ def keyword?; end
42
+ end
43
+ end
@@ -0,0 +1,7 @@
1
+ # typed: true
2
+
3
+ module Prism
4
+ # This module is responsible for converting the prism syntax tree into other
5
+ # syntax trees.
6
+ module Translation; end
7
+ end