prism 2.0.0-arm-linux-musl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. checksums.yaml +7 -0
  2. data/BSDmakefile +58 -0
  3. data/CHANGELOG.md +881 -0
  4. data/CODE_OF_CONDUCT.md +76 -0
  5. data/CONTRIBUTING.md +58 -0
  6. data/LICENSE.md +7 -0
  7. data/Makefile +145 -0
  8. data/README.md +81 -0
  9. data/config.yml +4752 -0
  10. data/docs/build_system.md +119 -0
  11. data/docs/configuration.md +69 -0
  12. data/docs/cruby_compilation.md +27 -0
  13. data/docs/design.md +53 -0
  14. data/docs/encoding.md +121 -0
  15. data/docs/fuzzing.md +81 -0
  16. data/docs/heredocs.md +36 -0
  17. data/docs/javascript.md +133 -0
  18. data/docs/local_variable_depth.md +229 -0
  19. data/docs/mapping.md +117 -0
  20. data/docs/parser_translation.md +24 -0
  21. data/docs/parsing_rules.md +22 -0
  22. data/docs/releasing.md +147 -0
  23. data/docs/relocation.md +32 -0
  24. data/docs/ripper_translation.md +63 -0
  25. data/docs/ruby_api.md +45 -0
  26. data/docs/ruby_parser_translation.md +19 -0
  27. data/docs/serialization.md +244 -0
  28. data/docs/testing.md +72 -0
  29. data/ext/prism/api_node.c +7005 -0
  30. data/ext/prism/extconf.rb +126 -0
  31. data/ext/prism/extension.c +1768 -0
  32. data/ext/prism/extension.h +19 -0
  33. data/include/prism/arena.h +45 -0
  34. data/include/prism/arena.hpp +42 -0
  35. data/include/prism/ast.h +10381 -0
  36. data/include/prism/ast.hpp +641 -0
  37. data/include/prism/buffer.h +60 -0
  38. data/include/prism/buffer.hpp +55 -0
  39. data/include/prism/comments.h +72 -0
  40. data/include/prism/compiler/accel.h +26 -0
  41. data/include/prism/compiler/align.h +36 -0
  42. data/include/prism/compiler/assume.h +25 -0
  43. data/include/prism/compiler/branch.h +26 -0
  44. data/include/prism/compiler/bswap.h +46 -0
  45. data/include/prism/compiler/cplusplus.hpp +16 -0
  46. data/include/prism/compiler/endianness.h +34 -0
  47. data/include/prism/compiler/exceptions.hpp +17 -0
  48. data/include/prism/compiler/exported.h +24 -0
  49. data/include/prism/compiler/fallthrough.h +22 -0
  50. data/include/prism/compiler/filesystem.h +32 -0
  51. data/include/prism/compiler/flex_array.h +19 -0
  52. data/include/prism/compiler/force_inline.h +21 -0
  53. data/include/prism/compiler/format.h +27 -0
  54. data/include/prism/compiler/inline.h +17 -0
  55. data/include/prism/compiler/lifetimebound.hpp +25 -0
  56. data/include/prism/compiler/nodiscard.h +22 -0
  57. data/include/prism/compiler/nonnull.h +18 -0
  58. data/include/prism/compiler/unused.h +18 -0
  59. data/include/prism/constant_pool.h +106 -0
  60. data/include/prism/diagnostic.h +111 -0
  61. data/include/prism/errors_format.h +61 -0
  62. data/include/prism/excludes.h +29 -0
  63. data/include/prism/integer.h +41 -0
  64. data/include/prism/internal/allocator.h +68 -0
  65. data/include/prism/internal/allocator_debug.h +88 -0
  66. data/include/prism/internal/arena.h +108 -0
  67. data/include/prism/internal/bit.h +42 -0
  68. data/include/prism/internal/buffer.h +94 -0
  69. data/include/prism/internal/char.h +145 -0
  70. data/include/prism/internal/comments.h +53 -0
  71. data/include/prism/internal/constant_pool.h +152 -0
  72. data/include/prism/internal/diagnostic.h +389 -0
  73. data/include/prism/internal/encoding.h +242 -0
  74. data/include/prism/internal/integer.h +68 -0
  75. data/include/prism/internal/isinf.h +16 -0
  76. data/include/prism/internal/line_offset_list.h +34 -0
  77. data/include/prism/internal/list.h +62 -0
  78. data/include/prism/internal/magic_comments.h +23 -0
  79. data/include/prism/internal/memchr.h +15 -0
  80. data/include/prism/internal/node.h +42 -0
  81. data/include/prism/internal/options.h +232 -0
  82. data/include/prism/internal/parser.h +1004 -0
  83. data/include/prism/internal/regexp.h +26 -0
  84. data/include/prism/internal/serialize.h +39 -0
  85. data/include/prism/internal/source.h +72 -0
  86. data/include/prism/internal/static_literals.h +100 -0
  87. data/include/prism/internal/stringy.h +30 -0
  88. data/include/prism/internal/strncasecmp.h +18 -0
  89. data/include/prism/internal/strpbrk.h +33 -0
  90. data/include/prism/internal/tokens.h +11 -0
  91. data/include/prism/internal/xxh3.h +16 -0
  92. data/include/prism/json.h +40 -0
  93. data/include/prism/line_offset_list.h +69 -0
  94. data/include/prism/magic_comments.h +53 -0
  95. data/include/prism/node.h +139 -0
  96. data/include/prism/node.hpp +272 -0
  97. data/include/prism/node_type.hpp +38 -0
  98. data/include/prism/options.h +363 -0
  99. data/include/prism/options.hpp +43 -0
  100. data/include/prism/parser.h +433 -0
  101. data/include/prism/parser.hpp +274 -0
  102. data/include/prism/prettyprint.h +39 -0
  103. data/include/prism/serialize.h +123 -0
  104. data/include/prism/source.h +163 -0
  105. data/include/prism/stream.h +36 -0
  106. data/include/prism/string_query.h +82 -0
  107. data/include/prism/stringy.h +61 -0
  108. data/include/prism/stringy.hpp +31 -0
  109. data/include/prism/version.h +46 -0
  110. data/include/prism.h +123 -0
  111. data/include/prism.hpp +24 -0
  112. data/lib/prism/3.3/prism.so +0 -0
  113. data/lib/prism/3.4/prism.so +0 -0
  114. data/lib/prism/4.0/prism.so +0 -0
  115. data/lib/prism/compiler.rb +815 -0
  116. data/lib/prism/desugar_compiler.rb +463 -0
  117. data/lib/prism/dispatcher.rb +2383 -0
  118. data/lib/prism/dot_visitor.rb +4848 -0
  119. data/lib/prism/dsl.rb +1369 -0
  120. data/lib/prism/ffi.rb +746 -0
  121. data/lib/prism/inspect_visitor.rb +2425 -0
  122. data/lib/prism/lex_compat.rb +953 -0
  123. data/lib/prism/mutation_compiler.rb +780 -0
  124. data/lib/prism/node.rb +33940 -0
  125. data/lib/prism/node_ext.rb +504 -0
  126. data/lib/prism/node_find.rb +215 -0
  127. data/lib/prism/parse_result/errors.rb +72 -0
  128. data/lib/prism/parse_result/newlines.rb +423 -0
  129. data/lib/prism/parse_result.rb +1236 -0
  130. data/lib/prism/pattern.rb +314 -0
  131. data/lib/prism/polyfill/append_as_bytes.rb +15 -0
  132. data/lib/prism/polyfill/byteindex.rb +13 -0
  133. data/lib/prism/polyfill/scan_byte.rb +14 -0
  134. data/lib/prism/polyfill/unpack1.rb +14 -0
  135. data/lib/prism/polyfill/warn.rb +36 -0
  136. data/lib/prism/reflection.rb +427 -0
  137. data/lib/prism/relocation.rb +567 -0
  138. data/lib/prism/serialize.rb +4589 -0
  139. data/lib/prism/string_query.rb +54 -0
  140. data/lib/prism/translation/parser/builder.rb +70 -0
  141. data/lib/prism/translation/parser/compiler.rb +2226 -0
  142. data/lib/prism/translation/parser/lexer.rb +769 -0
  143. data/lib/prism/translation/parser.rb +376 -0
  144. data/lib/prism/translation/parser_current.rb +26 -0
  145. data/lib/prism/translation/parser_versions.rb +36 -0
  146. data/lib/prism/translation/ripper/filter.rb +53 -0
  147. data/lib/prism/translation/ripper/lexer.rb +133 -0
  148. data/lib/prism/translation/ripper/sexp.rb +118 -0
  149. data/lib/prism/translation/ripper/shim.rb +7 -0
  150. data/lib/prism/translation/ripper.rb +4278 -0
  151. data/lib/prism/translation/ruby_parser.rb +1721 -0
  152. data/lib/prism/translation.rb +20 -0
  153. data/lib/prism/visitor.rb +1440 -0
  154. data/lib/prism.rb +168 -0
  155. data/prism.gemspec +251 -0
  156. data/rbi/generated/prism/compiler.rbi +489 -0
  157. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  158. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  159. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  160. data/rbi/generated/prism/dsl.rbi +751 -0
  161. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  162. data/rbi/generated/prism/lex_compat.rbi +162 -0
  163. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  164. data/rbi/generated/prism/node.rbi +15143 -0
  165. data/rbi/generated/prism/node_ext.rbi +259 -0
  166. data/rbi/generated/prism/node_find.rbi +79 -0
  167. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  168. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  169. data/rbi/generated/prism/parse_result.rbi +776 -0
  170. data/rbi/generated/prism/pattern.rbi +134 -0
  171. data/rbi/generated/prism/reflection.rbi +89 -0
  172. data/rbi/generated/prism/relocation.rbi +329 -0
  173. data/rbi/generated/prism/serialize.rbi +185 -0
  174. data/rbi/generated/prism/string_query.rbi +43 -0
  175. data/rbi/generated/prism/translation.rbi +7 -0
  176. data/rbi/generated/prism/visitor.rbi +650 -0
  177. data/rbi/generated/prism.rbi +86 -0
  178. data/rbi/prism/translation/parser.rbi +11 -0
  179. data/rbi/prism/translation/parser_versions.rbi +28 -0
  180. data/rbi/prism/translation/ripper.rbi +15 -0
  181. data/rbi/rubyvm/node_find.rbi +14 -0
  182. data/sig/generated/prism/compiler.rbs +492 -0
  183. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  184. data/sig/generated/prism/dispatcher.rbs +1292 -0
  185. data/sig/generated/prism/dot_visitor.rbs +635 -0
  186. data/sig/generated/prism/dsl.rbs +928 -0
  187. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  188. data/sig/generated/prism/lex_compat.rbs +166 -0
  189. data/sig/generated/prism/mutation_compiler.rbs +464 -0
  190. data/sig/generated/prism/node.rbs +21632 -0
  191. data/sig/generated/prism/node_ext.rbs +290 -0
  192. data/sig/generated/prism/node_find.rbs +90 -0
  193. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  194. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  195. data/sig/generated/prism/parse_result.rbs +871 -0
  196. data/sig/generated/prism/pattern.rbs +156 -0
  197. data/sig/generated/prism/reflection.rbs +101 -0
  198. data/sig/generated/prism/relocation.rbs +428 -0
  199. data/sig/generated/prism/serialize.rbs +200 -0
  200. data/sig/generated/prism/string_query.rbs +43 -0
  201. data/sig/generated/prism/translation.rbs +8 -0
  202. data/sig/generated/prism/visitor.rbs +1111 -0
  203. data/sig/generated/prism.rbs +84 -0
  204. data/sig/manifest.yaml +2 -0
  205. data/src/arena.c +117 -0
  206. data/src/buffer.c +389 -0
  207. data/src/char.c +289 -0
  208. data/src/comments.c +454 -0
  209. data/src/constant_pool.c +406 -0
  210. data/src/diagnostic.c +894 -0
  211. data/src/encoding.c +5345 -0
  212. data/src/errors_format.c +519 -0
  213. data/src/integer.c +681 -0
  214. data/src/json.c +5733 -0
  215. data/src/line_offset_list.c +100 -0
  216. data/src/list.c +24 -0
  217. data/src/memchr.c +37 -0
  218. data/src/node.c +4363 -0
  219. data/src/options.c +445 -0
  220. data/src/parser.c +371 -0
  221. data/src/prettyprint.c +9024 -0
  222. data/src/prism.c +23901 -0
  223. data/src/regexp.c +1705 -0
  224. data/src/serialize.c +2660 -0
  225. data/src/source.c +524 -0
  226. data/src/static_literals.c +685 -0
  227. data/src/string_query.c +180 -0
  228. data/src/stringy.c +61 -0
  229. data/src/strncasecmp.c +37 -0
  230. data/src/strpbrk.c +447 -0
  231. data/src/tokens.c +724 -0
  232. data/src/xxh3.c +292 -0
  233. metadata +276 -0
data/lib/prism/ffi.rb ADDED
@@ -0,0 +1,746 @@
1
+ # frozen_string_literal: true
2
+ # :markup: markdown
3
+ # typed: ignore
4
+
5
+ # This file is responsible for mirroring the API provided by the C extension by
6
+ # using FFI to call into the shared library.
7
+
8
+ require "rbconfig"
9
+ require "ffi"
10
+
11
+ # We want to eagerly load this file if there are Ractors so that it does not get
12
+ # autoloaded from within a non-main Ractor.
13
+ require "prism/serialize" if defined?(Ractor)
14
+
15
+ module Prism # :nodoc:
16
+ module LibRubyParser # :nodoc:
17
+ extend FFI::Library
18
+
19
+ # Define the library that we will be pulling functions from. Note that this
20
+ # must align with the build shared library from make/rake.
21
+ libprism_in_build = File.expand_path("../../build/libprism.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
22
+ libprism_in_libdir = "#{RbConfig::CONFIG["libdir"]}/prism/libprism.#{RbConfig::CONFIG["SOEXT"]}"
23
+
24
+ if File.exist?(libprism_in_build)
25
+ INCLUDE_DIR = File.expand_path("../../include", __dir__)
26
+ ffi_lib libprism_in_build
27
+ else
28
+ INCLUDE_DIR = "#{RbConfig::CONFIG["libdir"]}/prism/include"
29
+ ffi_lib libprism_in_libdir
30
+ end
31
+
32
+ # Convert a native C type declaration into a symbol that FFI understands.
33
+ # For example:
34
+ #
35
+ # const char * -> :pointer
36
+ # bool -> :bool
37
+ # size_t -> :size_t
38
+ # void -> :void
39
+ #
40
+ def self.resolve_type(type, callbacks)
41
+ type = type.strip
42
+
43
+ if !type.end_with?("*")
44
+ type.delete_prefix("const ").to_sym
45
+ else
46
+ type = type.delete_suffix("*").rstrip
47
+ callbacks.include?(type.to_sym) ? type.to_sym : :pointer
48
+ end
49
+ end
50
+
51
+ # Read through the given header file and find the declaration of each of the
52
+ # given functions. For each one, define a function with the same name and
53
+ # signature as the C function.
54
+ def self.load_exported_functions_from(header, *functions, callbacks)
55
+ File.foreach("#{INCLUDE_DIR}/#{header}") do |line|
56
+ # We only want to attempt to load exported functions.
57
+ next unless line.include?("PRISM_EXPORTED_FUNCTION ")
58
+
59
+ # We only want to load the functions that we are interested in.
60
+ next unless functions.any? { |function| line.include?(function) }
61
+
62
+ # Strip leading attributes (PRISM_EXPORTED_FUNCTION, PRISM_NODISCARD, etc.)
63
+ line = line.sub(/\A(PRISM_\w+(?:\([^)]*\))?)+/, "")
64
+ # Strip trailing attributes (PRISM_NODISCARD, PRISM_NONNULL(...), etc.)
65
+ line = line.sub(/\)(\s+PRISM_\w+(?:\([^)]*\))?)+\s*;/, ");")
66
+
67
+ # Parse the function declaration.
68
+ unless /^(?<return_type>.+) (?<name>\w+)\((?<arg_types>.+)\);$/ =~ line
69
+ raise "Could not parse #{line}"
70
+ end
71
+
72
+ # Delete the function from the list of functions we are looking for to
73
+ # mark it as having been found.
74
+ functions.delete(name)
75
+
76
+ # Split up the argument types into an array, ensure we handle the case
77
+ # where there are no arguments (by explicit void).
78
+ arg_types = arg_types.split(",").map(&:strip)
79
+ arg_types = [] if arg_types == %w[void]
80
+
81
+ # Resolve the type of the argument by dropping the name of the argument
82
+ # first if it is present.
83
+ arg_types.map! { |type| resolve_type(type.sub(/\w+$/, ""), callbacks) }
84
+
85
+ # Attach the function using the FFI library.
86
+ attach_function name, arg_types, resolve_type(return_type, [])
87
+ end
88
+
89
+ # If we didn't find all of the functions, raise an error.
90
+ raise "Could not find functions #{functions.inspect}" unless functions.empty?
91
+ end
92
+
93
+ callback :pm_source_stream_fgets_t, [:pointer, :int, :pointer], :pointer
94
+ callback :pm_source_stream_feof_t, [:pointer], :int
95
+ pm_source_init_result_values = %i[PM_SOURCE_INIT_SUCCESS PM_SOURCE_INIT_ERROR_GENERIC PM_SOURCE_INIT_ERROR_DIRECTORY PM_SOURCE_INIT_ERROR_NON_REGULAR]
96
+ enum :pm_source_init_result_t, pm_source_init_result_values
97
+ enum :pm_string_query_t, [:PM_STRING_QUERY_ERROR, -1, :PM_STRING_QUERY_FALSE, :PM_STRING_QUERY_TRUE]
98
+ enum :pm_errors_format_type_t, [:PM_ERRORS_FORMAT_PLAIN, 1, :PM_ERRORS_FORMAT_STYLE, :PM_ERRORS_FORMAT_COLOR]
99
+
100
+ # Ractor-safe lookup table for pm_source_init_result_t, since FFI's
101
+ # enum_type accesses module instance variables that are not shareable.
102
+ SOURCE_INIT_RESULT = pm_source_init_result_values.freeze
103
+
104
+ load_exported_functions_from(
105
+ "prism/version.h",
106
+ "pm_version",
107
+ []
108
+ )
109
+
110
+ load_exported_functions_from(
111
+ "prism/serialize.h",
112
+ "pm_serialize_parse",
113
+ "pm_serialize_parse_stream",
114
+ "pm_serialize_parse_comments",
115
+ "pm_serialize_lex",
116
+ "pm_serialize_parse_lex",
117
+ "pm_serialize_parse_success_p",
118
+ "pm_serialize_parse_errors_format",
119
+ []
120
+ )
121
+
122
+ load_exported_functions_from(
123
+ "prism/string_query.h",
124
+ "pm_string_query_local",
125
+ "pm_string_query_constant",
126
+ "pm_string_query_method_name",
127
+ "pm_string_query_keyword",
128
+ []
129
+ )
130
+
131
+ load_exported_functions_from(
132
+ "prism/buffer.h",
133
+ "pm_buffer_new",
134
+ "pm_buffer_value",
135
+ "pm_buffer_length",
136
+ "pm_buffer_free",
137
+ []
138
+ )
139
+
140
+ load_exported_functions_from(
141
+ "prism/source.h",
142
+ "pm_source_file_new",
143
+ "pm_source_mapped_new",
144
+ "pm_source_stream_new",
145
+ "pm_source_free",
146
+ "pm_source_source",
147
+ "pm_source_length",
148
+ [:pm_source_stream_fgets_t, :pm_source_stream_feof_t]
149
+ )
150
+
151
+ # This object represents a pm_buffer_t. We only use it as an opaque pointer,
152
+ # so it doesn't need to know the fields of pm_buffer_t.
153
+ class PrismBuffer # :nodoc:
154
+ attr_reader :pointer
155
+
156
+ def initialize(pointer)
157
+ @pointer = pointer
158
+ end
159
+
160
+ def value
161
+ LibRubyParser.pm_buffer_value(pointer)
162
+ end
163
+
164
+ def length
165
+ LibRubyParser.pm_buffer_length(pointer)
166
+ end
167
+
168
+ def read
169
+ value.read_string(length)
170
+ end
171
+
172
+ # Initialize a new buffer and yield it to the block. The buffer will be
173
+ # automatically freed when the block returns.
174
+ def self.with
175
+ buffer = LibRubyParser.pm_buffer_new
176
+ raise unless buffer
177
+
178
+ begin
179
+ yield new(buffer)
180
+ ensure
181
+ LibRubyParser.pm_buffer_free(buffer)
182
+ end
183
+ end
184
+ end
185
+
186
+ # This object represents source code to be parsed. For strings it wraps a
187
+ # pointer directly; for files it uses a pm_source_t under the hood.
188
+ class PrismSource # :nodoc:
189
+ PLATFORM_EXPECTS_UTF8 =
190
+ RbConfig::CONFIG["host_os"].match?(/cygwin|mingw|mswin|darwin/i)
191
+
192
+ attr_reader :pointer, :length
193
+
194
+ def initialize(pointer, length, from_string)
195
+ @pointer = pointer
196
+ @length = length
197
+ @from_string = from_string
198
+ end
199
+
200
+ def read
201
+ raise "should use the original String instead" if @from_string
202
+ @pointer.read_string(@length)
203
+ end
204
+
205
+ # Yields a PrismSource backed by the given string to the block.
206
+ def self.with_string(string)
207
+ raise TypeError unless string.is_a?(String)
208
+
209
+ length = string.bytesize
210
+ # + 1 to never get an address of 0, which pm_parser_init() asserts
211
+ FFI::MemoryPointer.new(:char, length + 1, false) do |pointer|
212
+ pointer.write_string(string)
213
+ # since we have the extra byte we might as well \0-terminate
214
+ pointer.put_char(length, 0)
215
+ return yield new(pointer, length, true)
216
+ end
217
+ end
218
+
219
+ # Yields a PrismSource to the given block, backed by a pm_source_t.
220
+ def self.with_file(filepath, options)
221
+ # File.path applies the same coercion and validation as the C
222
+ # extension's rb_get_path: it accepts to_path/to_str objects and
223
+ # rejects paths containing a NUL byte, which would otherwise truncate
224
+ # the path when it is handed to C as a NUL-terminated string.
225
+ filepath = File.path(filepath)
226
+ options[:filepath] = filepath
227
+
228
+ # On Windows and Mac, it's expected that filepaths will be encoded in
229
+ # UTF-8. If they are not, we need to convert them to UTF-8 before
230
+ # passing them into pm_source_mapped_new.
231
+ if PLATFORM_EXPECTS_UTF8 && (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
232
+ filepath = filepath.encode(Encoding::UTF_8)
233
+ end
234
+
235
+ FFI::MemoryPointer.new(:int) do |result_ptr|
236
+ pm_source = LibRubyParser.pm_source_mapped_new(filepath, 0, result_ptr)
237
+
238
+ case SOURCE_INIT_RESULT[result_ptr.read_int]
239
+ when :PM_SOURCE_INIT_SUCCESS
240
+ pointer = LibRubyParser.pm_source_source(pm_source)
241
+ length = LibRubyParser.pm_source_length(pm_source)
242
+ return yield new(pointer, length, false)
243
+ when :PM_SOURCE_INIT_ERROR_GENERIC
244
+ raise SystemCallError.new(filepath, FFI.errno)
245
+ when :PM_SOURCE_INIT_ERROR_DIRECTORY
246
+ raise Errno::EISDIR.new(filepath)
247
+ when :PM_SOURCE_INIT_ERROR_NON_REGULAR
248
+ # Fall back to reading the file through Ruby IO for non-regular
249
+ # files (pipes, character devices, etc.)
250
+ return with_string(File.read(filepath)) { |string| yield string }
251
+ else
252
+ raise "Unknown error initializing pm_source_t: #{result_ptr.read_int}"
253
+ end
254
+ ensure
255
+ LibRubyParser.pm_source_free(pm_source) if pm_source && !pm_source.null?
256
+ end
257
+ end
258
+ end
259
+ end
260
+
261
+ # Mark the LibRubyParser module as private as it should only be called through
262
+ # the prism module.
263
+ private_constant :LibRubyParser
264
+
265
+ # The version constant is set by reading the result of calling pm_version.
266
+ VERSION = LibRubyParser.pm_version.read_string.freeze
267
+
268
+ class << self
269
+ # Mirror the Prism.dump API by using the serialization API.
270
+ def dump(source, **options)
271
+ LibRubyParser::PrismSource.with_string(source) { |string| dump_common(string, options) }
272
+ end
273
+
274
+ # Mirror the Prism.dump_file API by using the serialization API.
275
+ def dump_file(filepath, **options)
276
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| dump_common(string, options) }
277
+ end
278
+
279
+ # Mirror the Prism.lex API by using the serialization API.
280
+ def lex(code, **options)
281
+ LibRubyParser::PrismSource.with_string(code) { |string| lex_common(string, code, options) }
282
+ end
283
+
284
+ # Mirror the Prism.lex_file API by using the serialization API.
285
+ def lex_file(filepath, **options)
286
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| lex_common(string, string.read, options) }
287
+ end
288
+
289
+ # Mirror the Prism.parse API by using the serialization API.
290
+ def parse(code, **options)
291
+ LibRubyParser::PrismSource.with_string(code) { |string| parse_common(string, code, options) }
292
+ end
293
+
294
+ # Mirror the Prism.parse_file API by using the serialization API. This uses
295
+ # native strings instead of Ruby strings because it allows us to use mmap
296
+ # when it is available.
297
+ def parse_file(filepath, **options)
298
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_common(string, string.read, options) }
299
+ end
300
+
301
+ # Mirror the Prism.parse_stream API by using the serialization API.
302
+ def parse_stream(stream, **options)
303
+ format_type = raise_error_format_type(options)
304
+
305
+ LibRubyParser::PrismBuffer.with do |buffer|
306
+ # The largest number of bytes a single character can occupy in any
307
+ # encoding Ruby supports. IO#gets(limit) may return up to
308
+ # (max_enc_len - 1) bytes more than requested to avoid splitting a
309
+ # multi-byte character, so we reserve that much headroom (plus the NUL
310
+ # terminator) to guarantee the result fits in the caller's buffer. This
311
+ # mirrors MAX_ENC_LEN in ext/prism/extension.c.
312
+ max_enc_len = 6
313
+
314
+ source = +""
315
+ callback = -> (string, size, _) {
316
+ raise "Expected size to be > #{max_enc_len}, got: #{size}" if size <= max_enc_len
317
+
318
+ line = String.try_convert(stream.gets(size - max_enc_len))
319
+ if !line.nil?
320
+ # A misbehaving `gets` may ignore the limit; never write past the
321
+ # buffer (one byte is reserved for the NUL terminator).
322
+ line = line.byteslice(0, size - 1) if line.bytesize > size - 1
323
+ source << line
324
+ string.write_string("#{line}\x00", line.bytesize + 1)
325
+ end
326
+ }
327
+
328
+ eof_callback = -> (_) { stream.eof? }
329
+
330
+ pm_source = LibRubyParser.pm_source_stream_new(nil, callback, eof_callback)
331
+ begin
332
+ LibRubyParser.pm_serialize_parse_stream(buffer.pointer, pm_source, dump_options(options))
333
+ result = Prism.load(source, buffer.read, options.fetch(:freeze, false))
334
+
335
+ if format_type && result.failure?
336
+ LibRubyParser::PrismSource.with_string(source) do |string|
337
+ raise_error(string, options, format_type)
338
+ end
339
+ end
340
+
341
+ result
342
+ ensure
343
+ LibRubyParser.pm_source_free(pm_source) if pm_source && !pm_source.null?
344
+ end
345
+ end
346
+ end
347
+
348
+ # Mirror the Prism.parse_comments API by using the serialization API.
349
+ def parse_comments(code, **options)
350
+ LibRubyParser::PrismSource.with_string(code) { |string| parse_comments_common(string, code, options) }
351
+ end
352
+
353
+ # Mirror the Prism.parse_file_comments API by using the serialization
354
+ # API. This uses native strings instead of Ruby strings because it allows us
355
+ # to use mmap when it is available.
356
+ def parse_file_comments(filepath, **options)
357
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_comments_common(string, string.read, options) }
358
+ end
359
+
360
+ # Mirror the Prism.parse_lex API by using the serialization API.
361
+ def parse_lex(code, **options)
362
+ LibRubyParser::PrismSource.with_string(code) { |string| parse_lex_common(string, code, options) }
363
+ end
364
+
365
+ # Mirror the Prism.parse_lex_file API by using the serialization API.
366
+ def parse_lex_file(filepath, **options)
367
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_lex_common(string, string.read, options) }
368
+ end
369
+
370
+ # Mirror the Prism.parse_success? API by using the serialization API.
371
+ def parse_success?(code, **options)
372
+ LibRubyParser::PrismSource.with_string(code) { |string| parse_file_success_common(string, options) }
373
+ end
374
+
375
+ # Mirror the Prism.parse_failure? API by using the serialization API.
376
+ def parse_failure?(code, **options)
377
+ !parse_success?(code, **options)
378
+ end
379
+
380
+ # Mirror the Prism.parse_file_success? API by using the serialization API.
381
+ def parse_file_success?(filepath, **options)
382
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_file_success_common(string, options) }
383
+ end
384
+
385
+ # Mirror the Prism.parse_file_failure? API by using the serialization API.
386
+ def parse_file_failure?(filepath, **options)
387
+ !parse_file_success?(filepath, **options)
388
+ end
389
+
390
+ # Mirror the Prism.profile API by using the serialization API.
391
+ def profile(source, **options)
392
+ LibRubyParser::PrismSource.with_string(source) do |string|
393
+ if (format_type = raise_error_format_type(options))
394
+ raise_error(string, options, format_type)
395
+ end
396
+
397
+ LibRubyParser::PrismBuffer.with do |buffer|
398
+ LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
399
+ nil
400
+ end
401
+ end
402
+ end
403
+
404
+ # Mirror the Prism.profile_file API by using the serialization API.
405
+ def profile_file(filepath, **options)
406
+ LibRubyParser::PrismSource.with_file(filepath, options) do |string|
407
+ if (format_type = raise_error_format_type(options))
408
+ raise_error(string, options, format_type)
409
+ end
410
+
411
+ LibRubyParser::PrismBuffer.with do |buffer|
412
+ LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
413
+ nil
414
+ end
415
+ end
416
+ end
417
+
418
+ private
419
+
420
+ def dump_common(string, options) # :nodoc:
421
+ if (format_type = raise_error_format_type(options))
422
+ raise_error(string, options, format_type)
423
+ end
424
+
425
+ LibRubyParser::PrismBuffer.with do |buffer|
426
+ LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
427
+
428
+ dumped = buffer.read
429
+ dumped.freeze if options.fetch(:freeze, false)
430
+
431
+ dumped
432
+ end
433
+ end
434
+
435
+ def lex_common(string, code, options) # :nodoc:
436
+ format_type = raise_error_format_type(options)
437
+
438
+ LibRubyParser::PrismBuffer.with do |buffer|
439
+ LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
440
+ result = Serialize.load_lex(code, buffer.read, options.fetch(:freeze, false))
441
+
442
+ raise_error(string, options, format_type) if format_type && result.failure?
443
+ result
444
+ end
445
+ end
446
+
447
+ def parse_common(string, code, options) # :nodoc:
448
+ format_type = raise_error_format_type(options)
449
+ serialized = dump_common(string, options)
450
+ result = Serialize.load_parse(code, serialized, options.fetch(:freeze, false))
451
+
452
+ raise_error(string, options, format_type) if format_type && result.failure?
453
+ result
454
+ end
455
+
456
+ def parse_comments_common(string, code, options) # :nodoc:
457
+ if (format_type = raise_error_format_type(options))
458
+ raise_error(string, options, format_type)
459
+ end
460
+
461
+ LibRubyParser::PrismBuffer.with do |buffer|
462
+ LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options))
463
+ Serialize.load_parse_comments(code, buffer.read, options.fetch(:freeze, false))
464
+ end
465
+ end
466
+
467
+ def parse_lex_common(string, code, options) # :nodoc:
468
+ format_type = raise_error_format_type(options)
469
+
470
+ LibRubyParser::PrismBuffer.with do |buffer|
471
+ LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
472
+ result = Serialize.load_parse_lex(code, buffer.read, options.fetch(:freeze, false))
473
+
474
+ raise_error(string, options, format_type) if format_type && result.failure?
475
+ result
476
+ end
477
+ end
478
+
479
+ def parse_file_success_common(string, options) # :nodoc:
480
+ format_type = raise_error_format_type(options)
481
+ success = LibRubyParser.pm_serialize_parse_success_p(string.pointer, string.length, dump_options(options))
482
+
483
+ raise_error(string, options, format_type) if format_type && !success
484
+ success
485
+ end
486
+
487
+ # Extract the raise_error option from the given options hash and convert
488
+ # it into the format type that should be used when formatting errors, or
489
+ # nil if raising is disabled.
490
+ def raise_error_format_type(options) # :nodoc:
491
+ case (value = options.delete(:raise_error))
492
+ when nil, false
493
+ nil
494
+ when true
495
+ # When given true, mirror the behavior of CRuby itself: color when
496
+ # $stderr is a terminal (unless NO_COLOR is set), bold styling when
497
+ # NO_COLOR is set, plain otherwise. This policy is duplicated in
498
+ # ext/prism/extension.c (build_options_i) and the two must stay in
499
+ # sync.
500
+ if $stderr.respond_to?(:tty?) && $stderr.tty?
501
+ no_color = ENV["NO_COLOR"]
502
+ (no_color.nil? || no_color.empty?) ? :PM_ERRORS_FORMAT_COLOR : :PM_ERRORS_FORMAT_STYLE
503
+ else
504
+ :PM_ERRORS_FORMAT_PLAIN
505
+ end
506
+ when :plain
507
+ :PM_ERRORS_FORMAT_PLAIN
508
+ when :style
509
+ :PM_ERRORS_FORMAT_STYLE
510
+ when :color
511
+ :PM_ERRORS_FORMAT_COLOR
512
+ when Symbol
513
+ raise ArgumentError, "invalid raise_error value: #{value}"
514
+ else
515
+ raise TypeError, "wrong argument type #{value.class} (expected Symbol)"
516
+ end
517
+ end
518
+
519
+ # Parse the given source and format any errors that are encountered into
520
+ # an appropriate exception, then raise it. If the source parses without
521
+ # any errors, then return nil.
522
+ def raise_error(string, options, format_type) # :nodoc:
523
+ LibRubyParser::PrismBuffer.with do |buffer|
524
+ level = LibRubyParser.pm_serialize_parse_errors_format(buffer.pointer, string.pointer, string.length, dump_options(options), format_type)
525
+ return if level == -1
526
+
527
+ encoding_name, _, message = buffer.read.partition("\0")
528
+
529
+ case level
530
+ when 0 # syntax
531
+ error = SyntaxError.new(message.force_encoding(encoding_name))
532
+ error.instance_variable_set(:@path, options.fetch(:filepath, ""))
533
+ raise error
534
+ when 1 # argument
535
+ raise ArgumentError, message.force_encoding(encoding_name)
536
+ when 2 # load
537
+ error = LoadError.new(message.force_encoding(Encoding.find("locale")))
538
+ error.instance_variable_set(:@path, nil)
539
+ raise error
540
+ else
541
+ raise "Unknown error level: #{level}"
542
+ end
543
+ end
544
+ end
545
+
546
+ # Return the value that should be dumped for the command_line option.
547
+ def dump_options_command_line(options)
548
+ command_line = options.fetch(:command_line, "")
549
+ raise ArgumentError, "command_line must be a string" unless command_line.is_a?(String)
550
+
551
+ command_line.each_char.inject(0) do |value, char|
552
+ case char
553
+ when "a" then value | 0b000001
554
+ when "e" then value | 0b000010
555
+ when "l" then value | 0b000100
556
+ when "n" then value | 0b001000
557
+ when "p" then value | 0b010000
558
+ when "x" then value | 0b100000
559
+ else raise ArgumentError, "invalid command_line option: #{char}"
560
+ end
561
+ end
562
+ end
563
+
564
+ # Return the value that should be dumped for the version option.
565
+ def dump_options_version(version)
566
+ case version
567
+ when "current"
568
+ version_string_to_number(RUBY_VERSION) || raise(CurrentVersionError, RUBY_VERSION)
569
+ when "latest"
570
+ 0 # Handled in pm_parser_init
571
+ when "nearest", nil
572
+ dump = version_string_to_number(RUBY_VERSION)
573
+ return dump if dump
574
+ if RUBY_VERSION < "3.3"
575
+ version_string_to_number("3.3")
576
+ else
577
+ 0 # Handled in pm_parser_init
578
+ end
579
+ else
580
+ version_string_to_number(version) || raise(ArgumentError, "invalid version: #{version}")
581
+ end
582
+ end
583
+
584
+ # Converts a version string like "4.0.0" or "4.0" into a number.
585
+ # Returns nil if the version is unknown.
586
+ def version_string_to_number(version)
587
+ case version
588
+ when /\A3\.3(\.\d+)?\z/
589
+ 1
590
+ when /\A3\.4(\.\d+)?\z/
591
+ 2
592
+ when /\A3\.5(\.\d+)?\z/, /\A4\.0(\.\d+)?\z/
593
+ 3
594
+ when /\A4\.1(\.\d+)?\z/
595
+ 4
596
+ end
597
+ end
598
+
599
+ # The set of options that are understood by the parsing APIs. Note that
600
+ # raise_error is not listed here because it is deleted from the options
601
+ # hash by raise_error_format_type before the options are dumped.
602
+ DUMP_OPTIONS_KEYS = [:attach_comments, :command_line, :encoding, :filepath, :freeze, :frozen_string_literal, :line, :main_script, :partial_script, :scopes, :version].freeze
603
+ private_constant :DUMP_OPTIONS_KEYS
604
+
605
+ # Convert the given options into a serialized options string.
606
+ def dump_options(options)
607
+ unknown_keys = options.keys - DUMP_OPTIONS_KEYS
608
+ raise ArgumentError, "unknown keyword: #{unknown_keys.first}" unless unknown_keys.empty?
609
+
610
+ template = +""
611
+ values = []
612
+
613
+ template << "L"
614
+ if (filepath = options[:filepath])
615
+ values.push(filepath.bytesize, filepath.b)
616
+ template << "A*"
617
+ else
618
+ values << 0
619
+ end
620
+
621
+ template << "l"
622
+ values << options.fetch(:line, 1)
623
+
624
+ template << "L"
625
+ if (encoding = options[:encoding])
626
+ name = encoding.is_a?(Encoding) ? encoding.name : encoding
627
+ values.push(name.bytesize, name.b)
628
+ template << "A*"
629
+ else
630
+ values << 0
631
+ end
632
+
633
+ template << "C"
634
+ values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
635
+
636
+ template << "C"
637
+ values << dump_options_command_line(options)
638
+
639
+ template << "C"
640
+ values << dump_options_version(options[:version])
641
+
642
+ template << "C"
643
+ values << (options[:encoding] == false ? 1 : 0)
644
+
645
+ template << "C"
646
+ values << (options.fetch(:main_script, false) ? 1 : 0)
647
+
648
+ template << "C"
649
+ values << (options.fetch(:partial_script, false) ? 1 : 0)
650
+
651
+ template << "C"
652
+ values << (options.fetch(:freeze, false) ? 1 : 0)
653
+
654
+ template << "C"
655
+ values << (options.fetch(:attach_comments, false) ? 1 : 0)
656
+
657
+ template << "L"
658
+ if (scopes = options[:scopes])
659
+ values << scopes.length
660
+
661
+ scopes.each do |scope|
662
+ locals = nil
663
+ forwarding = 0
664
+
665
+ case scope
666
+ when Array
667
+ locals = scope
668
+ when Scope
669
+ locals = scope.locals
670
+
671
+ scope.forwarding.each do |forward|
672
+ case forward
673
+ when :* then forwarding |= 0x1
674
+ when :** then forwarding |= 0x2
675
+ when :& then forwarding |= 0x4
676
+ when :"..." then forwarding |= 0x8
677
+ else raise ArgumentError, "invalid forwarding value: #{forward}"
678
+ end
679
+ end
680
+ else
681
+ raise TypeError, "wrong argument type #{scope.class.inspect} (expected Array or Prism::Scope)"
682
+ end
683
+
684
+ template << "L"
685
+ values << locals.length
686
+
687
+ template << "C"
688
+ values << forwarding
689
+
690
+ locals.each do |local|
691
+ name = local.name
692
+ template << "L"
693
+ values << name.bytesize
694
+
695
+ template << "A*"
696
+ values << name.b
697
+ end
698
+ end
699
+ else
700
+ values << 0
701
+ end
702
+
703
+ values.pack(template)
704
+ end
705
+ end
706
+
707
+ # Here we are going to patch StringQuery to put in the class-level methods so
708
+ # that it can maintain a consistent interface
709
+ class StringQuery # :nodoc:
710
+ class << self
711
+ # Mirrors the C extension's StringQuery::local? method.
712
+ def local?(string)
713
+ query(LibRubyParser.pm_string_query_local(string, string.bytesize, string.encoding.name))
714
+ end
715
+
716
+ # Mirrors the C extension's StringQuery::constant? method.
717
+ def constant?(string)
718
+ query(LibRubyParser.pm_string_query_constant(string, string.bytesize, string.encoding.name))
719
+ end
720
+
721
+ # Mirrors the C extension's StringQuery::method_name? method.
722
+ def method_name?(string)
723
+ query(LibRubyParser.pm_string_query_method_name(string, string.bytesize, string.encoding.name))
724
+ end
725
+
726
+ # Mirrors the C extension's StringQuery::keyword? method.
727
+ def keyword?(string)
728
+ query(LibRubyParser.pm_string_query_keyword(string, string.bytesize, string.encoding.name))
729
+ end
730
+
731
+ private
732
+
733
+ # Parse the enum result and return an appropriate boolean.
734
+ def query(result)
735
+ case result
736
+ when :PM_STRING_QUERY_ERROR
737
+ raise ArgumentError, "Invalid or non ascii-compatible encoding"
738
+ when :PM_STRING_QUERY_FALSE
739
+ false
740
+ when :PM_STRING_QUERY_TRUE
741
+ true
742
+ end
743
+ end
744
+ end
745
+ end
746
+ end