prism 1.9.0 → 2.0.0.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +41 -12
  6. data/README.md +35 -97
  7. data/config.yml +174 -161
  8. data/docs/build_system.md +2 -2
  9. data/docs/configuration.md +5 -4
  10. data/docs/encoding.md +2 -2
  11. data/docs/fuzzing.md +4 -11
  12. data/docs/javascript.md +19 -4
  13. data/docs/mapping.md +1 -1
  14. data/docs/releasing.md +77 -5
  15. data/docs/relocation.md +2 -4
  16. data/docs/serialization.md +22 -24
  17. data/docs/testing.md +46 -29
  18. data/ext/prism/api_node.c +1461 -1401
  19. data/ext/prism/extconf.rb +3 -4
  20. data/ext/prism/extension.c +796 -422
  21. data/ext/prism/extension.h +3 -3
  22. data/include/prism/arena.h +37 -0
  23. data/include/prism/ast.h +2727 -610
  24. data/include/prism/buffer.h +52 -0
  25. data/include/prism/comments.h +55 -0
  26. data/include/prism/compiler/accel.h +26 -0
  27. data/include/prism/compiler/align.h +36 -0
  28. data/include/prism/compiler/assume.h +25 -0
  29. data/include/prism/compiler/bswap.h +46 -0
  30. data/include/prism/compiler/endianness.h +34 -0
  31. data/include/prism/compiler/exported.h +24 -0
  32. data/include/prism/compiler/fallthrough.h +22 -0
  33. data/include/prism/compiler/filesystem.h +32 -0
  34. data/include/prism/compiler/flex_array.h +19 -0
  35. data/include/prism/compiler/force_inline.h +21 -0
  36. data/include/prism/compiler/format.h +25 -0
  37. data/include/prism/compiler/inline.h +17 -0
  38. data/include/prism/compiler/nodiscard.h +22 -0
  39. data/include/prism/compiler/nonnull.h +18 -0
  40. data/include/prism/compiler/unused.h +18 -0
  41. data/include/prism/constant_pool.h +81 -0
  42. data/include/prism/diagnostic.h +38 -403
  43. data/include/prism/errors_format.h +53 -0
  44. data/include/prism/excludes.h +29 -0
  45. data/include/prism/integer.h +41 -0
  46. data/include/prism/internal/allocator.h +68 -0
  47. data/include/prism/internal/allocator_debug.h +88 -0
  48. data/include/prism/internal/arena.h +108 -0
  49. data/include/prism/internal/bit.h +42 -0
  50. data/include/prism/internal/buffer.h +94 -0
  51. data/include/prism/internal/char.h +145 -0
  52. data/include/prism/internal/comments.h +53 -0
  53. data/include/prism/internal/constant_pool.h +152 -0
  54. data/include/prism/internal/diagnostic.h +389 -0
  55. data/include/prism/{encoding.h → internal/encoding.h} +27 -68
  56. data/include/prism/internal/integer.h +68 -0
  57. data/include/prism/internal/isinf.h +16 -0
  58. data/include/prism/internal/line_offset_list.h +34 -0
  59. data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  60. data/include/prism/internal/magic_comments.h +23 -0
  61. data/include/prism/internal/memchr.h +15 -0
  62. data/include/prism/internal/node.h +38 -0
  63. data/include/prism/internal/options.h +232 -0
  64. data/include/prism/internal/parser.h +1004 -0
  65. data/include/prism/internal/regexp.h +26 -0
  66. data/include/prism/internal/serialize.h +39 -0
  67. data/include/prism/internal/source.h +72 -0
  68. data/include/prism/{static_literals.h → internal/static_literals.h} +26 -47
  69. data/include/prism/internal/stringy.h +35 -0
  70. data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  71. data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  72. data/include/prism/internal/tokens.h +11 -0
  73. data/include/prism/internal/xxh3.h +16 -0
  74. data/include/prism/json.h +32 -0
  75. data/include/prism/line_offset_list.h +61 -0
  76. data/include/prism/magic_comments.h +35 -0
  77. data/include/prism/node.h +18 -53
  78. data/include/prism/options.h +146 -280
  79. data/include/prism/parser.h +356 -867
  80. data/include/prism/prettyprint.h +6 -9
  81. data/include/prism/serialize.h +115 -0
  82. data/include/prism/source.h +155 -0
  83. data/include/prism/stream.h +28 -0
  84. data/include/prism/string_query.h +74 -0
  85. data/include/prism/stringy.h +72 -0
  86. data/include/prism/version.h +12 -3
  87. data/include/prism.h +43 -320
  88. data/lib/prism/compiler.rb +319 -305
  89. data/lib/prism/desugar_compiler.rb +105 -34
  90. data/lib/prism/dispatcher.rb +646 -473
  91. data/lib/prism/dot_visitor.rb +592 -511
  92. data/lib/prism/dsl.rb +391 -25
  93. data/lib/prism/ffi.rb +258 -90
  94. data/lib/prism/inspect_visitor.rb +357 -325
  95. data/lib/prism/lex_compat.rb +254 -212
  96. data/lib/prism/mutation_compiler.rb +313 -305
  97. data/lib/prism/node.rb +20966 -6992
  98. data/lib/prism/node_ext.rb +180 -187
  99. data/lib/prism/node_find.rb +215 -0
  100. data/lib/prism/parse_result/errors.rb +7 -1
  101. data/lib/prism/parse_result/newlines.rb +276 -8
  102. data/lib/prism/parse_result.rb +468 -139
  103. data/lib/prism/pattern.rb +65 -20
  104. data/lib/prism/reflection.rb +23 -12
  105. data/lib/prism/relocation.rb +149 -87
  106. data/lib/prism/serialize.rb +3460 -539
  107. data/lib/prism/string_query.rb +24 -1
  108. data/lib/prism/translation/parser/builder.rb +10 -2
  109. data/lib/prism/translation/parser/compiler.rb +44 -52
  110. data/lib/prism/translation/parser/lexer.rb +41 -92
  111. data/lib/prism/translation/parser.rb +1 -1
  112. data/lib/prism/translation/parser_current.rb +1 -1
  113. data/lib/prism/translation/ripper/lexer.rb +8 -10
  114. data/lib/prism/translation/ripper/sexp.rb +2 -10
  115. data/lib/prism/translation/ripper/shim.rb +2 -0
  116. data/lib/prism/translation/ripper.rb +967 -209
  117. data/lib/prism/translation/ruby_parser.rb +73 -316
  118. data/lib/prism/translation.rb +2 -0
  119. data/lib/prism/visitor.rb +632 -5
  120. data/lib/prism.rb +79 -9
  121. data/prism.gemspec +128 -62
  122. data/rbi/generated/prism/compiler.rbi +489 -0
  123. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  124. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  125. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  126. data/rbi/generated/prism/dsl.rbi +751 -0
  127. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  128. data/rbi/generated/prism/lex_compat.rbi +162 -0
  129. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  130. data/rbi/generated/prism/node.rbi +14407 -0
  131. data/rbi/generated/prism/node_ext.rbi +259 -0
  132. data/rbi/generated/prism/node_find.rbi +79 -0
  133. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  134. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  135. data/rbi/generated/prism/parse_result.rbi +776 -0
  136. data/rbi/generated/prism/pattern.rbi +134 -0
  137. data/rbi/generated/prism/reflection.rbi +89 -0
  138. data/rbi/generated/prism/relocation.rbi +329 -0
  139. data/rbi/generated/prism/serialize.rbi +184 -0
  140. data/rbi/generated/prism/string_query.rbi +43 -0
  141. data/rbi/generated/prism/translation.rbi +7 -0
  142. data/rbi/generated/prism/visitor.rbi +650 -0
  143. data/rbi/generated/prism.rbi +86 -0
  144. data/rbi/prism/translation/parser_versions.rbi +5 -0
  145. data/rbi/rubyvm/node_find.rbi +14 -0
  146. data/sig/generated/prism/compiler.rbs +492 -0
  147. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  148. data/sig/generated/prism/dispatcher.rbs +1292 -0
  149. data/sig/generated/prism/dot_visitor.rbs +635 -0
  150. data/sig/generated/prism/dsl.rbs +928 -0
  151. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  152. data/sig/generated/prism/lex_compat.rbs +166 -0
  153. data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
  154. data/sig/generated/prism/node.rbs +20896 -0
  155. data/sig/generated/prism/node_ext.rbs +290 -0
  156. data/sig/generated/prism/node_find.rbs +90 -0
  157. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  158. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  159. data/sig/generated/prism/parse_result.rbs +871 -0
  160. data/sig/generated/prism/pattern.rbs +156 -0
  161. data/sig/generated/prism/reflection.rbs +101 -0
  162. data/sig/generated/prism/relocation.rbs +428 -0
  163. data/sig/generated/prism/serialize.rbs +198 -0
  164. data/sig/generated/prism/string_query.rbs +43 -0
  165. data/sig/generated/prism/translation.rbs +8 -0
  166. data/sig/generated/prism/visitor.rbs +1111 -0
  167. data/sig/generated/prism.rbs +84 -0
  168. data/sig/manifest.yaml +2 -0
  169. data/src/arena.c +117 -0
  170. data/src/{util/pm_buffer.c → buffer.c} +53 -21
  171. data/src/{util/pm_char.c → char.c} +28 -57
  172. data/src/comments.c +454 -0
  173. data/src/constant_pool.c +406 -0
  174. data/src/diagnostic.c +97 -65
  175. data/src/encoding.c +8 -3
  176. data/src/errors_format.c +519 -0
  177. data/src/{util/pm_integer.c → integer.c} +33 -22
  178. data/src/json.c +5733 -0
  179. data/src/line_offset_list.c +100 -0
  180. data/src/list.c +24 -0
  181. data/src/{util/pm_memchr.c → memchr.c} +8 -6
  182. data/src/node.c +3504 -7674
  183. data/src/options.c +162 -49
  184. data/src/parser.c +367 -0
  185. data/src/prettyprint.c +421 -354
  186. data/src/prism.c +8402 -7161
  187. data/src/regexp.c +961 -45
  188. data/src/serialize.c +460 -424
  189. data/src/source.c +518 -0
  190. data/src/static_literals.c +111 -43
  191. data/src/string_query.c +180 -0
  192. data/src/stringy.c +91 -0
  193. data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  194. data/src/strpbrk.c +447 -0
  195. data/src/{token_type.c → tokens.c} +39 -18
  196. data/src/xxh3.c +292 -0
  197. metadata +128 -63
  198. data/ext/prism/api_pack.c +0 -276
  199. data/include/prism/defines.h +0 -260
  200. data/include/prism/pack.h +0 -163
  201. data/include/prism/regexp.h +0 -43
  202. data/include/prism/util/pm_buffer.h +0 -236
  203. data/include/prism/util/pm_char.h +0 -204
  204. data/include/prism/util/pm_constant_pool.h +0 -218
  205. data/include/prism/util/pm_integer.h +0 -130
  206. data/include/prism/util/pm_memchr.h +0 -29
  207. data/include/prism/util/pm_newline_list.h +0 -113
  208. data/include/prism/util/pm_string.h +0 -200
  209. data/lib/prism/pack.rb +0 -230
  210. data/lib/prism/parse_result/comments.rb +0 -188
  211. data/rbi/prism/compiler.rbi +0 -12
  212. data/rbi/prism/dsl.rbi +0 -524
  213. data/rbi/prism/inspect_visitor.rbi +0 -12
  214. data/rbi/prism/node.rbi +0 -8750
  215. data/rbi/prism/node_ext.rbi +0 -107
  216. data/rbi/prism/parse_result.rbi +0 -404
  217. data/rbi/prism/reflection.rbi +0 -58
  218. data/rbi/prism/string_query.rbi +0 -12
  219. data/rbi/prism/visitor.rbi +0 -473
  220. data/rbi/prism.rbi +0 -63
  221. data/sig/prism/compiler.rbs +0 -9
  222. data/sig/prism/dispatcher.rbs +0 -19
  223. data/sig/prism/dot_visitor.rbs +0 -6
  224. data/sig/prism/dsl.rbs +0 -351
  225. data/sig/prism/inspect_visitor.rbs +0 -22
  226. data/sig/prism/lex_compat.rbs +0 -10
  227. data/sig/prism/node.rbs +0 -4042
  228. data/sig/prism/node_ext.rbs +0 -149
  229. data/sig/prism/pack.rbs +0 -43
  230. data/sig/prism/parse_result/comments.rbs +0 -38
  231. data/sig/prism/parse_result.rbs +0 -197
  232. data/sig/prism/pattern.rbs +0 -13
  233. data/sig/prism/reflection.rbs +0 -50
  234. data/sig/prism/relocation.rbs +0 -185
  235. data/sig/prism/serialize.rbs +0 -8
  236. data/sig/prism/string_query.rbs +0 -11
  237. data/sig/prism/visitor.rbs +0 -169
  238. data/sig/prism.rbs +0 -272
  239. data/src/pack.c +0 -509
  240. data/src/util/pm_constant_pool.c +0 -342
  241. data/src/util/pm_list.c +0 -49
  242. data/src/util/pm_newline_list.c +0 -125
  243. data/src/util/pm_string.c +0 -381
  244. data/src/util/pm_strpbrk.c +0 -206
@@ -0,0 +1,198 @@
1
+ # Generated from lib/prism/serialize.rb with RBS::Inline
2
+
3
+ module Prism
4
+ # A module responsible for deserializing parse results.
5
+ module Serialize
6
+ # The major version of prism that we are expecting to find in the serialized
7
+ # strings.
8
+ MAJOR_VERSION: ::Integer
9
+
10
+ # The minor version of prism that we are expecting to find in the serialized
11
+ # strings.
12
+ MINOR_VERSION: ::Integer
13
+
14
+ # The patch version of prism that we are expecting to find in the serialized
15
+ # strings.
16
+ PATCH_VERSION: ::Integer
17
+
18
+ # Deserialize the dumped output from a request to parse or parse_file.
19
+ #
20
+ # The formatting of the source of this method is purposeful to illustrate
21
+ # the structure of the serialized data.
22
+ #
23
+ # : (String input, String serialized, bool freeze) -> ParseResult
24
+ def self.load_parse: (String input, String serialized, bool freeze) -> ParseResult
25
+
26
+ # Deserialize the dumped output from a request to lex or lex_file.
27
+ #
28
+ # The formatting of the source of this method is purposeful to illustrate
29
+ # the structure of the serialized data.
30
+ #
31
+ # : (String input, String serialized, bool freeze) -> LexResult
32
+ def self.load_lex: (String input, String serialized, bool freeze) -> LexResult
33
+
34
+ # Deserialize the dumped output from a request to parse_comments or
35
+ # parse_file_comments.
36
+ #
37
+ # The formatting of the source of this method is purposeful to illustrate
38
+ # the structure of the serialized data.
39
+ #
40
+ # : (String input, String serialized, bool freeze) -> Array[Comment]
41
+ def self.load_parse_comments: (String input, String serialized, bool freeze) -> Array[Comment]
42
+
43
+ # Deserialize the dumped output from a request to parse_lex or
44
+ # parse_lex_file.
45
+ #
46
+ # The formatting of the source of this method is purposeful to illustrate
47
+ # the structure of the serialized data.
48
+ #
49
+ # : (String input, String serialized, bool freeze) -> ParseLexResult
50
+ def self.load_parse_lex: (String input, String serialized, bool freeze) -> ParseLexResult
51
+
52
+ class ConstantPool
53
+ # :nodoc:
54
+ attr_reader size: Integer
55
+
56
+ @serialized: String
57
+
58
+ @base: Integer
59
+
60
+ @pool: Array[Symbol?]
61
+
62
+ # : (String serialized, Integer base, Integer size) -> void
63
+ def initialize: (String serialized, Integer base, Integer size) -> void
64
+
65
+ # : (Integer index, Encoding encoding) -> Symbol
66
+ def get: (Integer index, Encoding encoding) -> Symbol
67
+ end
68
+
69
+ FastStringIO: untyped
70
+
71
+ class Loader
72
+ # :nodoc:
73
+ attr_reader input: String
74
+
75
+ attr_reader io: StringIO
76
+
77
+ attr_reader source: Source
78
+
79
+ # : (Source source, String serialized) -> void
80
+ def initialize: (Source source, String serialized) -> void
81
+
82
+ # : () -> bool
83
+ def eof?: () -> bool
84
+
85
+ # : (ConstantPool constant_pool) -> void
86
+ def load_constant_pool: (ConstantPool constant_pool) -> void
87
+
88
+ # : () -> void
89
+ def load_header: () -> void
90
+
91
+ # : () -> Encoding
92
+ def load_encoding: () -> Encoding
93
+
94
+ # : (bool freeze) -> Array[Integer]
95
+ def load_line_offsets: (bool freeze) -> Array[Integer]
96
+
97
+ # Load the comments attached to each node into the comment attachments and
98
+ # comment ranges of the given source. For each node that owns comments, the
99
+ # index at which its comments begin is stored at its node id and the index
100
+ # at which they end at the following node id.
101
+ #
102
+ # : (Source source, bool freeze) -> void
103
+ def load_comment_attachments: (Source source, bool freeze) -> void
104
+
105
+ # Load the comments into the given list, which belongs to the source that
106
+ # the comments refer to.
107
+ #
108
+ # : (Array[Comment] comments, bool freeze) -> Array[Comment]
109
+ def load_comments: (Array[Comment] comments, bool freeze) -> Array[Comment]
110
+
111
+ # : (bool freeze) -> Array[MagicComment]
112
+ def load_magic_comments: (bool freeze) -> Array[MagicComment]
113
+
114
+ DIAGNOSTIC_TYPES: Array[Symbol]
115
+
116
+ # : () -> Symbol
117
+ def load_error_level: () -> Symbol
118
+
119
+ # : (Encoding encoding, bool freeze) -> Array[ParseError]
120
+ def load_errors: (Encoding encoding, bool freeze) -> Array[ParseError]
121
+
122
+ # : () -> Symbol
123
+ def load_warning_level: () -> Symbol
124
+
125
+ # : (Encoding encoding, bool freeze) -> Array[ParseWarning]
126
+ def load_warnings: (Encoding encoding, bool freeze) -> Array[ParseWarning]
127
+
128
+ # : () -> Array[Token]
129
+ def load_tokens: () -> Array[Token]
130
+
131
+ # variable-length integer using https://en.wikipedia.org/wiki/LEB128
132
+ # This is also what protobuf uses: https://protobuf.dev/programming-guides/encoding/#varints
133
+ #
134
+ # : () -> Integer
135
+ def load_varuint: () -> Integer
136
+
137
+ # : () -> Integer
138
+ def load_varsint: () -> Integer
139
+
140
+ # : () -> Integer
141
+ def load_integer: () -> Integer
142
+
143
+ # : () -> Float
144
+ def load_double: () -> Float
145
+
146
+ # : () -> bool
147
+ def load_bool: () -> bool
148
+
149
+ # : () -> Integer
150
+ def load_uint32: () -> Integer
151
+
152
+ # : () -> Integer
153
+ def load_uint64: () -> Integer
154
+
155
+ # : (ConstantPool constant_pool, Encoding encoding, bool freeze) -> node?
156
+ def load_optional_node: (ConstantPool constant_pool, Encoding encoding, bool freeze) -> node?
157
+
158
+ # : (Encoding encoding) -> String
159
+ def load_string: (Encoding encoding) -> String
160
+
161
+ # : (bool freeze) -> Location
162
+ def load_location_object: (bool freeze) -> Location
163
+
164
+ # Load a location object from the serialized data. Note that we are lying
165
+ # about the signature a bit here, because we sometimes load it as a packed
166
+ # integer instead of an object.
167
+ #
168
+ # : (bool freeze) -> Location
169
+ def load_location: (bool freeze) -> Location
170
+
171
+ # Load an optional location object from the serialized data if it is
172
+ # present. Note that we are lying about the signature a bit here, because
173
+ # we sometimes load it as a packed integer instead of an object.
174
+ #
175
+ # : (bool freeze) -> Location?
176
+ def load_optional_location: (bool freeze) -> Location?
177
+
178
+ # : (bool freeze) -> Location?
179
+ def load_optional_location_object: (bool freeze) -> Location?
180
+
181
+ # : (ConstantPool constant_pool, Encoding encoding) -> Symbol
182
+ def load_constant: (ConstantPool constant_pool, Encoding encoding) -> Symbol
183
+
184
+ # : (ConstantPool constant_pool, Encoding encoding) -> Symbol?
185
+ def load_optional_constant: (ConstantPool constant_pool, Encoding encoding) -> Symbol?
186
+
187
+ # : (ConstantPool constant_pool, Encoding encoding, bool freeze) -> node
188
+ def load_node: (ConstantPool constant_pool, Encoding encoding, bool freeze) -> node
189
+
190
+ @load_node_lambdas: Array[Proc]
191
+
192
+ def define_load_node_lambdas: () -> void
193
+ end
194
+
195
+ # The token types that can be indexed by their enum values.
196
+ TOKEN_TYPES: Array[Symbol?]
197
+ end
198
+ end
@@ -0,0 +1,43 @@
1
+ # Generated from lib/prism/string_query.rb with RBS::Inline
2
+
3
+ module Prism
4
+ # Query methods that allow categorizing strings based on their context for
5
+ # where they could be valid in a Ruby syntax tree.
6
+ class StringQuery
7
+ def self.local?: (String string) -> bool
8
+
9
+ def self.constant?: (String string) -> bool
10
+
11
+ def self.method_name?: (String string) -> bool
12
+
13
+ def self.keyword?: (String string) -> bool
14
+
15
+ # The string that this query is wrapping.
16
+ attr_reader string: String
17
+
18
+ # Initialize a new query with the given string.
19
+ #
20
+ # : (String string) -> void
21
+ def initialize: (String string) -> void
22
+
23
+ # Whether or not this string is a valid local variable name.
24
+ #
25
+ # : () -> bool
26
+ def local?: () -> bool
27
+
28
+ # Whether or not this string is a valid constant name.
29
+ #
30
+ # : () -> bool
31
+ def constant?: () -> bool
32
+
33
+ # Whether or not this string is a valid method name.
34
+ #
35
+ # : () -> bool
36
+ def method_name?: () -> bool
37
+
38
+ # Whether or not this string is a valid keyword.
39
+ #
40
+ # : () -> bool
41
+ def keyword?: () -> bool
42
+ end
43
+ end
@@ -0,0 +1,8 @@
1
+ # Generated from lib/prism/translation.rb with RBS::Inline
2
+
3
+ module Prism
4
+ # This module is responsible for converting the prism syntax tree into other
5
+ # syntax trees.
6
+ module Translation
7
+ end
8
+ end