prism 2.0.0.rc1-x86_64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. checksums.yaml +7 -0
  2. data/BSDmakefile +58 -0
  3. data/CHANGELOG.md +870 -0
  4. data/CODE_OF_CONDUCT.md +76 -0
  5. data/CONTRIBUTING.md +58 -0
  6. data/LICENSE.md +7 -0
  7. data/Makefile +145 -0
  8. data/README.md +81 -0
  9. data/config.yml +4752 -0
  10. data/docs/build_system.md +119 -0
  11. data/docs/configuration.md +69 -0
  12. data/docs/cruby_compilation.md +27 -0
  13. data/docs/design.md +53 -0
  14. data/docs/encoding.md +121 -0
  15. data/docs/fuzzing.md +81 -0
  16. data/docs/heredocs.md +36 -0
  17. data/docs/javascript.md +133 -0
  18. data/docs/local_variable_depth.md +229 -0
  19. data/docs/mapping.md +117 -0
  20. data/docs/parser_translation.md +24 -0
  21. data/docs/parsing_rules.md +22 -0
  22. data/docs/releasing.md +147 -0
  23. data/docs/relocation.md +32 -0
  24. data/docs/ripper_translation.md +63 -0
  25. data/docs/ruby_api.md +45 -0
  26. data/docs/ruby_parser_translation.md +19 -0
  27. data/docs/serialization.md +230 -0
  28. data/docs/testing.md +72 -0
  29. data/ext/prism/api_node.c +7005 -0
  30. data/ext/prism/extconf.rb +126 -0
  31. data/ext/prism/extension.c +1783 -0
  32. data/ext/prism/extension.h +19 -0
  33. data/include/prism/arena.h +37 -0
  34. data/include/prism/ast.h +10371 -0
  35. data/include/prism/buffer.h +52 -0
  36. data/include/prism/comments.h +55 -0
  37. data/include/prism/compiler/accel.h +24 -0
  38. data/include/prism/compiler/align.h +36 -0
  39. data/include/prism/compiler/assume.h +25 -0
  40. data/include/prism/compiler/exported.h +24 -0
  41. data/include/prism/compiler/fallthrough.h +22 -0
  42. data/include/prism/compiler/filesystem.h +32 -0
  43. data/include/prism/compiler/flex_array.h +19 -0
  44. data/include/prism/compiler/force_inline.h +21 -0
  45. data/include/prism/compiler/format.h +25 -0
  46. data/include/prism/compiler/inline.h +17 -0
  47. data/include/prism/compiler/nodiscard.h +22 -0
  48. data/include/prism/compiler/nonnull.h +18 -0
  49. data/include/prism/compiler/unused.h +18 -0
  50. data/include/prism/constant_pool.h +81 -0
  51. data/include/prism/diagnostic.h +93 -0
  52. data/include/prism/errors_format.h +53 -0
  53. data/include/prism/excludes.h +29 -0
  54. data/include/prism/integer.h +41 -0
  55. data/include/prism/internal/allocator.h +68 -0
  56. data/include/prism/internal/allocator_debug.h +88 -0
  57. data/include/prism/internal/arena.h +108 -0
  58. data/include/prism/internal/bit.h +42 -0
  59. data/include/prism/internal/buffer.h +91 -0
  60. data/include/prism/internal/char.h +145 -0
  61. data/include/prism/internal/comments.h +53 -0
  62. data/include/prism/internal/constant_pool.h +152 -0
  63. data/include/prism/internal/diagnostic.h +389 -0
  64. data/include/prism/internal/encoding.h +242 -0
  65. data/include/prism/internal/integer.h +68 -0
  66. data/include/prism/internal/isinf.h +16 -0
  67. data/include/prism/internal/line_offset_list.h +34 -0
  68. data/include/prism/internal/list.h +62 -0
  69. data/include/prism/internal/magic_comments.h +23 -0
  70. data/include/prism/internal/memchr.h +15 -0
  71. data/include/prism/internal/node.h +38 -0
  72. data/include/prism/internal/options.h +232 -0
  73. data/include/prism/internal/parser.h +984 -0
  74. data/include/prism/internal/regexp.h +26 -0
  75. data/include/prism/internal/serialize.h +39 -0
  76. data/include/prism/internal/source.h +72 -0
  77. data/include/prism/internal/static_literals.h +100 -0
  78. data/include/prism/internal/stringy.h +35 -0
  79. data/include/prism/internal/strncasecmp.h +18 -0
  80. data/include/prism/internal/strpbrk.h +33 -0
  81. data/include/prism/internal/tokens.h +11 -0
  82. data/include/prism/json.h +32 -0
  83. data/include/prism/line_offset_list.h +61 -0
  84. data/include/prism/magic_comments.h +35 -0
  85. data/include/prism/node.h +94 -0
  86. data/include/prism/options.h +354 -0
  87. data/include/prism/parser.h +404 -0
  88. data/include/prism/prettyprint.h +31 -0
  89. data/include/prism/serialize.h +115 -0
  90. data/include/prism/source.h +155 -0
  91. data/include/prism/stream.h +28 -0
  92. data/include/prism/string_query.h +63 -0
  93. data/include/prism/stringy.h +72 -0
  94. data/include/prism/version.h +38 -0
  95. data/include/prism.h +131 -0
  96. data/lib/prism/3.3/prism.so +0 -0
  97. data/lib/prism/3.4/prism.so +0 -0
  98. data/lib/prism/4.0/prism.so +0 -0
  99. data/lib/prism/compiler.rb +815 -0
  100. data/lib/prism/desugar_compiler.rb +463 -0
  101. data/lib/prism/dispatcher.rb +2383 -0
  102. data/lib/prism/dot_visitor.rb +4848 -0
  103. data/lib/prism/dsl.rb +1369 -0
  104. data/lib/prism/ffi.rb +740 -0
  105. data/lib/prism/inspect_visitor.rb +2425 -0
  106. data/lib/prism/lex_compat.rb +953 -0
  107. data/lib/prism/mutation_compiler.rb +780 -0
  108. data/lib/prism/node.rb +33204 -0
  109. data/lib/prism/node_ext.rb +504 -0
  110. data/lib/prism/node_find.rb +215 -0
  111. data/lib/prism/parse_result/errors.rb +72 -0
  112. data/lib/prism/parse_result/newlines.rb +423 -0
  113. data/lib/prism/parse_result.rb +1221 -0
  114. data/lib/prism/pattern.rb +314 -0
  115. data/lib/prism/polyfill/append_as_bytes.rb +15 -0
  116. data/lib/prism/polyfill/byteindex.rb +13 -0
  117. data/lib/prism/polyfill/scan_byte.rb +14 -0
  118. data/lib/prism/polyfill/unpack1.rb +14 -0
  119. data/lib/prism/polyfill/warn.rb +36 -0
  120. data/lib/prism/reflection.rb +427 -0
  121. data/lib/prism/relocation.rb +567 -0
  122. data/lib/prism/serialize.rb +5308 -0
  123. data/lib/prism/string_query.rb +46 -0
  124. data/lib/prism/translation/parser/builder.rb +70 -0
  125. data/lib/prism/translation/parser/compiler.rb +2226 -0
  126. data/lib/prism/translation/parser/lexer.rb +769 -0
  127. data/lib/prism/translation/parser.rb +376 -0
  128. data/lib/prism/translation/parser_current.rb +26 -0
  129. data/lib/prism/translation/parser_versions.rb +36 -0
  130. data/lib/prism/translation/ripper/filter.rb +53 -0
  131. data/lib/prism/translation/ripper/lexer.rb +133 -0
  132. data/lib/prism/translation/ripper/sexp.rb +118 -0
  133. data/lib/prism/translation/ripper/shim.rb +7 -0
  134. data/lib/prism/translation/ripper.rb +4278 -0
  135. data/lib/prism/translation/ruby_parser.rb +1721 -0
  136. data/lib/prism/translation.rb +20 -0
  137. data/lib/prism/visitor.rb +1440 -0
  138. data/lib/prism.rb +168 -0
  139. data/prism.gemspec +234 -0
  140. data/rbi/generated/prism/compiler.rbi +489 -0
  141. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  142. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  143. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  144. data/rbi/generated/prism/dsl.rbi +751 -0
  145. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  146. data/rbi/generated/prism/lex_compat.rbi +162 -0
  147. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  148. data/rbi/generated/prism/node.rbi +13681 -0
  149. data/rbi/generated/prism/node_ext.rbi +259 -0
  150. data/rbi/generated/prism/node_find.rbi +79 -0
  151. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  152. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  153. data/rbi/generated/prism/parse_result.rbi +764 -0
  154. data/rbi/generated/prism/pattern.rbi +134 -0
  155. data/rbi/generated/prism/reflection.rbi +89 -0
  156. data/rbi/generated/prism/relocation.rbi +329 -0
  157. data/rbi/generated/prism/serialize.rbi +181 -0
  158. data/rbi/generated/prism/string_query.rbi +36 -0
  159. data/rbi/generated/prism/translation.rbi +7 -0
  160. data/rbi/generated/prism/visitor.rbi +650 -0
  161. data/rbi/generated/prism.rbi +86 -0
  162. data/rbi/prism/translation/parser.rbi +11 -0
  163. data/rbi/prism/translation/parser_versions.rbi +28 -0
  164. data/rbi/prism/translation/ripper.rbi +15 -0
  165. data/rbi/rubyvm/node_find.rbi +14 -0
  166. data/sig/generated/prism/compiler.rbs +492 -0
  167. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  168. data/sig/generated/prism/dispatcher.rbs +1292 -0
  169. data/sig/generated/prism/dot_visitor.rbs +635 -0
  170. data/sig/generated/prism/dsl.rbs +928 -0
  171. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  172. data/sig/generated/prism/lex_compat.rbs +166 -0
  173. data/sig/generated/prism/mutation_compiler.rbs +464 -0
  174. data/sig/generated/prism/node.rbs +20160 -0
  175. data/sig/generated/prism/node_ext.rbs +290 -0
  176. data/sig/generated/prism/node_find.rbs +90 -0
  177. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  178. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  179. data/sig/generated/prism/parse_result.rbs +859 -0
  180. data/sig/generated/prism/pattern.rbs +156 -0
  181. data/sig/generated/prism/reflection.rbs +101 -0
  182. data/sig/generated/prism/relocation.rbs +428 -0
  183. data/sig/generated/prism/serialize.rbs +195 -0
  184. data/sig/generated/prism/string_query.rbs +36 -0
  185. data/sig/generated/prism/translation.rbs +8 -0
  186. data/sig/generated/prism/visitor.rbs +1111 -0
  187. data/sig/generated/prism.rbs +84 -0
  188. data/sig/manifest.yaml +2 -0
  189. data/src/arena.c +117 -0
  190. data/src/buffer.c +380 -0
  191. data/src/char.c +289 -0
  192. data/src/comments.c +454 -0
  193. data/src/constant_pool.c +406 -0
  194. data/src/diagnostic.c +886 -0
  195. data/src/encoding.c +5345 -0
  196. data/src/errors_format.c +519 -0
  197. data/src/integer.c +681 -0
  198. data/src/json.c +5733 -0
  199. data/src/line_offset_list.c +100 -0
  200. data/src/list.c +24 -0
  201. data/src/memchr.c +37 -0
  202. data/src/node.c +4515 -0
  203. data/src/options.c +451 -0
  204. data/src/parser.c +349 -0
  205. data/src/prettyprint.c +9024 -0
  206. data/src/prism.c +23739 -0
  207. data/src/regexp.c +1706 -0
  208. data/src/serialize.c +2309 -0
  209. data/src/source.c +518 -0
  210. data/src/static_literals.c +685 -0
  211. data/src/string_query.c +166 -0
  212. data/src/stringy.c +91 -0
  213. data/src/strncasecmp.c +37 -0
  214. data/src/strpbrk.c +447 -0
  215. data/src/tokens.c +724 -0
  216. metadata +259 -0
data/lib/prism/ffi.rb ADDED
@@ -0,0 +1,740 @@
1
+ # frozen_string_literal: true
2
+ # :markup: markdown
3
+ # typed: ignore
4
+
5
+ # This file is responsible for mirroring the API provided by the C extension by
6
+ # using FFI to call into the shared library.
7
+
8
+ require "rbconfig"
9
+ require "ffi"
10
+
11
+ # We want to eagerly load this file if there are Ractors so that it does not get
12
+ # autoloaded from within a non-main Ractor.
13
+ require "prism/serialize" if defined?(Ractor)
14
+
15
+ module Prism # :nodoc:
16
+ module LibRubyParser # :nodoc:
17
+ extend FFI::Library
18
+
19
+ # Define the library that we will be pulling functions from. Note that this
20
+ # must align with the build shared library from make/rake.
21
+ libprism_in_build = File.expand_path("../../build/libprism.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
22
+ libprism_in_libdir = "#{RbConfig::CONFIG["libdir"]}/prism/libprism.#{RbConfig::CONFIG["SOEXT"]}"
23
+
24
+ if File.exist?(libprism_in_build)
25
+ INCLUDE_DIR = File.expand_path("../../include", __dir__)
26
+ ffi_lib libprism_in_build
27
+ else
28
+ INCLUDE_DIR = "#{RbConfig::CONFIG["libdir"]}/prism/include"
29
+ ffi_lib libprism_in_libdir
30
+ end
31
+
32
+ # Convert a native C type declaration into a symbol that FFI understands.
33
+ # For example:
34
+ #
35
+ # const char * -> :pointer
36
+ # bool -> :bool
37
+ # size_t -> :size_t
38
+ # void -> :void
39
+ #
40
+ def self.resolve_type(type, callbacks)
41
+ type = type.strip
42
+
43
+ if !type.end_with?("*")
44
+ type.delete_prefix("const ").to_sym
45
+ else
46
+ type = type.delete_suffix("*").rstrip
47
+ callbacks.include?(type.to_sym) ? type.to_sym : :pointer
48
+ end
49
+ end
50
+
51
+ # Read through the given header file and find the declaration of each of the
52
+ # given functions. For each one, define a function with the same name and
53
+ # signature as the C function.
54
+ def self.load_exported_functions_from(header, *functions, callbacks)
55
+ File.foreach("#{INCLUDE_DIR}/#{header}") do |line|
56
+ # We only want to attempt to load exported functions.
57
+ next unless line.include?("PRISM_EXPORTED_FUNCTION ")
58
+
59
+ # We only want to load the functions that we are interested in.
60
+ next unless functions.any? { |function| line.include?(function) }
61
+
62
+ # Strip leading attributes (PRISM_EXPORTED_FUNCTION, PRISM_NODISCARD, etc.)
63
+ line = line.sub(/\A(PRISM_\w+(?:\([^)]*\))?)+/, "")
64
+ # Strip trailing attributes (PRISM_NODISCARD, PRISM_NONNULL(...), etc.)
65
+ line = line.sub(/\)(\s+PRISM_\w+(?:\([^)]*\))?)+\s*;/, ");")
66
+
67
+ # Parse the function declaration.
68
+ unless /^(?<return_type>.+) (?<name>\w+)\((?<arg_types>.+)\);$/ =~ line
69
+ raise "Could not parse #{line}"
70
+ end
71
+
72
+ # Delete the function from the list of functions we are looking for to
73
+ # mark it as having been found.
74
+ functions.delete(name)
75
+
76
+ # Split up the argument types into an array, ensure we handle the case
77
+ # where there are no arguments (by explicit void).
78
+ arg_types = arg_types.split(",").map(&:strip)
79
+ arg_types = [] if arg_types == %w[void]
80
+
81
+ # Resolve the type of the argument by dropping the name of the argument
82
+ # first if it is present.
83
+ arg_types.map! { |type| resolve_type(type.sub(/\w+$/, ""), callbacks) }
84
+
85
+ # Attach the function using the FFI library.
86
+ attach_function name, arg_types, resolve_type(return_type, [])
87
+ end
88
+
89
+ # If we didn't find all of the functions, raise an error.
90
+ raise "Could not find functions #{functions.inspect}" unless functions.empty?
91
+ end
92
+
93
+ callback :pm_source_stream_fgets_t, [:pointer, :int, :pointer], :pointer
94
+ callback :pm_source_stream_feof_t, [:pointer], :int
95
+ pm_source_init_result_values = %i[PM_SOURCE_INIT_SUCCESS PM_SOURCE_INIT_ERROR_GENERIC PM_SOURCE_INIT_ERROR_DIRECTORY PM_SOURCE_INIT_ERROR_NON_REGULAR]
96
+ enum :pm_source_init_result_t, pm_source_init_result_values
97
+ enum :pm_string_query_t, [:PM_STRING_QUERY_ERROR, -1, :PM_STRING_QUERY_FALSE, :PM_STRING_QUERY_TRUE]
98
+ enum :pm_errors_format_type_t, [:PM_ERRORS_FORMAT_PLAIN, 1, :PM_ERRORS_FORMAT_STYLE, :PM_ERRORS_FORMAT_COLOR]
99
+
100
+ # Ractor-safe lookup table for pm_source_init_result_t, since FFI's
101
+ # enum_type accesses module instance variables that are not shareable.
102
+ SOURCE_INIT_RESULT = pm_source_init_result_values.freeze
103
+
104
+ load_exported_functions_from(
105
+ "prism/version.h",
106
+ "pm_version",
107
+ []
108
+ )
109
+
110
+ load_exported_functions_from(
111
+ "prism/serialize.h",
112
+ "pm_serialize_parse",
113
+ "pm_serialize_parse_stream",
114
+ "pm_serialize_parse_comments",
115
+ "pm_serialize_lex",
116
+ "pm_serialize_parse_lex",
117
+ "pm_serialize_parse_success_p",
118
+ "pm_serialize_parse_errors_format",
119
+ []
120
+ )
121
+
122
+ load_exported_functions_from(
123
+ "prism/string_query.h",
124
+ "pm_string_query_local",
125
+ "pm_string_query_constant",
126
+ "pm_string_query_method_name",
127
+ []
128
+ )
129
+
130
+ load_exported_functions_from(
131
+ "prism/buffer.h",
132
+ "pm_buffer_new",
133
+ "pm_buffer_value",
134
+ "pm_buffer_length",
135
+ "pm_buffer_free",
136
+ []
137
+ )
138
+
139
+ load_exported_functions_from(
140
+ "prism/source.h",
141
+ "pm_source_file_new",
142
+ "pm_source_mapped_new",
143
+ "pm_source_stream_new",
144
+ "pm_source_free",
145
+ "pm_source_source",
146
+ "pm_source_length",
147
+ [:pm_source_stream_fgets_t, :pm_source_stream_feof_t]
148
+ )
149
+
150
+ # This object represents a pm_buffer_t. We only use it as an opaque pointer,
151
+ # so it doesn't need to know the fields of pm_buffer_t.
152
+ class PrismBuffer # :nodoc:
153
+ attr_reader :pointer
154
+
155
+ def initialize(pointer)
156
+ @pointer = pointer
157
+ end
158
+
159
+ def value
160
+ LibRubyParser.pm_buffer_value(pointer)
161
+ end
162
+
163
+ def length
164
+ LibRubyParser.pm_buffer_length(pointer)
165
+ end
166
+
167
+ def read
168
+ value.read_string(length)
169
+ end
170
+
171
+ # Initialize a new buffer and yield it to the block. The buffer will be
172
+ # automatically freed when the block returns.
173
+ def self.with
174
+ buffer = LibRubyParser.pm_buffer_new
175
+ raise unless buffer
176
+
177
+ begin
178
+ yield new(buffer)
179
+ ensure
180
+ LibRubyParser.pm_buffer_free(buffer)
181
+ end
182
+ end
183
+ end
184
+
185
+ # This object represents source code to be parsed. For strings it wraps a
186
+ # pointer directly; for files it uses a pm_source_t under the hood.
187
+ class PrismSource # :nodoc:
188
+ PLATFORM_EXPECTS_UTF8 =
189
+ RbConfig::CONFIG["host_os"].match?(/cygwin|mingw|mswin|darwin/i)
190
+
191
+ attr_reader :pointer, :length
192
+
193
+ def initialize(pointer, length, from_string)
194
+ @pointer = pointer
195
+ @length = length
196
+ @from_string = from_string
197
+ end
198
+
199
+ def read
200
+ raise "should use the original String instead" if @from_string
201
+ @pointer.read_string(@length)
202
+ end
203
+
204
+ # Yields a PrismSource backed by the given string to the block.
205
+ def self.with_string(string)
206
+ raise TypeError unless string.is_a?(String)
207
+
208
+ length = string.bytesize
209
+ # + 1 to never get an address of 0, which pm_parser_init() asserts
210
+ FFI::MemoryPointer.new(:char, length + 1, false) do |pointer|
211
+ pointer.write_string(string)
212
+ # since we have the extra byte we might as well \0-terminate
213
+ pointer.put_char(length, 0)
214
+ return yield new(pointer, length, true)
215
+ end
216
+ end
217
+
218
+ # Yields a PrismSource to the given block, backed by a pm_source_t.
219
+ def self.with_file(filepath, options)
220
+ # File.path applies the same coercion and validation as the C
221
+ # extension's rb_get_path: it accepts to_path/to_str objects and
222
+ # rejects paths containing a NUL byte, which would otherwise truncate
223
+ # the path when it is handed to C as a NUL-terminated string.
224
+ filepath = File.path(filepath)
225
+ options[:filepath] = filepath
226
+
227
+ # On Windows and Mac, it's expected that filepaths will be encoded in
228
+ # UTF-8. If they are not, we need to convert them to UTF-8 before
229
+ # passing them into pm_source_mapped_new.
230
+ if PLATFORM_EXPECTS_UTF8 && (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
231
+ filepath = filepath.encode(Encoding::UTF_8)
232
+ end
233
+
234
+ FFI::MemoryPointer.new(:int) do |result_ptr|
235
+ pm_source = LibRubyParser.pm_source_mapped_new(filepath, 0, result_ptr)
236
+
237
+ case SOURCE_INIT_RESULT[result_ptr.read_int]
238
+ when :PM_SOURCE_INIT_SUCCESS
239
+ pointer = LibRubyParser.pm_source_source(pm_source)
240
+ length = LibRubyParser.pm_source_length(pm_source)
241
+ return yield new(pointer, length, false)
242
+ when :PM_SOURCE_INIT_ERROR_GENERIC
243
+ raise SystemCallError.new(filepath, FFI.errno)
244
+ when :PM_SOURCE_INIT_ERROR_DIRECTORY
245
+ raise Errno::EISDIR.new(filepath)
246
+ when :PM_SOURCE_INIT_ERROR_NON_REGULAR
247
+ # Fall back to reading the file through Ruby IO for non-regular
248
+ # files (pipes, character devices, etc.)
249
+ return with_string(File.read(filepath)) { |string| yield string }
250
+ else
251
+ raise "Unknown error initializing pm_source_t: #{result_ptr.read_int}"
252
+ end
253
+ ensure
254
+ LibRubyParser.pm_source_free(pm_source) if pm_source && !pm_source.null?
255
+ end
256
+ end
257
+ end
258
+ end
259
+
260
+ # Mark the LibRubyParser module as private as it should only be called through
261
+ # the prism module.
262
+ private_constant :LibRubyParser
263
+
264
+ # The version constant is set by reading the result of calling pm_version.
265
+ VERSION = LibRubyParser.pm_version.read_string.freeze
266
+
267
+ class << self
268
+ # Mirror the Prism.dump API by using the serialization API.
269
+ def dump(source, **options)
270
+ LibRubyParser::PrismSource.with_string(source) { |string| dump_common(string, options) }
271
+ end
272
+
273
+ # Mirror the Prism.dump_file API by using the serialization API.
274
+ def dump_file(filepath, **options)
275
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| dump_common(string, options) }
276
+ end
277
+
278
+ # Mirror the Prism.lex API by using the serialization API.
279
+ def lex(code, **options)
280
+ LibRubyParser::PrismSource.with_string(code) { |string| lex_common(string, code, options) }
281
+ end
282
+
283
+ # Mirror the Prism.lex_file API by using the serialization API.
284
+ def lex_file(filepath, **options)
285
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| lex_common(string, string.read, options) }
286
+ end
287
+
288
+ # Mirror the Prism.parse API by using the serialization API.
289
+ def parse(code, **options)
290
+ LibRubyParser::PrismSource.with_string(code) { |string| parse_common(string, code, options) }
291
+ end
292
+
293
+ # Mirror the Prism.parse_file API by using the serialization API. This uses
294
+ # native strings instead of Ruby strings because it allows us to use mmap
295
+ # when it is available.
296
+ def parse_file(filepath, **options)
297
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_common(string, string.read, options) }
298
+ end
299
+
300
+ # Mirror the Prism.parse_stream API by using the serialization API.
301
+ def parse_stream(stream, **options)
302
+ format_type = raise_error_format_type(options)
303
+
304
+ LibRubyParser::PrismBuffer.with do |buffer|
305
+ # The largest number of bytes a single character can occupy in any
306
+ # encoding Ruby supports. IO#gets(limit) may return up to
307
+ # (max_enc_len - 1) bytes more than requested to avoid splitting a
308
+ # multi-byte character, so we reserve that much headroom (plus the NUL
309
+ # terminator) to guarantee the result fits in the caller's buffer. This
310
+ # mirrors MAX_ENC_LEN in ext/prism/extension.c.
311
+ max_enc_len = 6
312
+
313
+ source = +""
314
+ callback = -> (string, size, _) {
315
+ raise "Expected size to be > #{max_enc_len}, got: #{size}" if size <= max_enc_len
316
+
317
+ line = String.try_convert(stream.gets(size - max_enc_len))
318
+ if !line.nil?
319
+ # A misbehaving `gets` may ignore the limit; never write past the
320
+ # buffer (one byte is reserved for the NUL terminator).
321
+ line = line.byteslice(0, size - 1) if line.bytesize > size - 1
322
+ source << line
323
+ string.write_string("#{line}\x00", line.bytesize + 1)
324
+ end
325
+ }
326
+
327
+ eof_callback = -> (_) { stream.eof? }
328
+
329
+ pm_source = LibRubyParser.pm_source_stream_new(nil, callback, eof_callback)
330
+ begin
331
+ LibRubyParser.pm_serialize_parse_stream(buffer.pointer, pm_source, dump_options(options))
332
+ result = Prism.load(source, buffer.read, options.fetch(:freeze, false))
333
+
334
+ if format_type && result.failure?
335
+ LibRubyParser::PrismSource.with_string(source) do |string|
336
+ raise_error(string, options, format_type)
337
+ end
338
+ end
339
+
340
+ result
341
+ ensure
342
+ LibRubyParser.pm_source_free(pm_source) if pm_source && !pm_source.null?
343
+ end
344
+ end
345
+ end
346
+
347
+ # Mirror the Prism.parse_comments API by using the serialization API.
348
+ def parse_comments(code, **options)
349
+ LibRubyParser::PrismSource.with_string(code) { |string| parse_comments_common(string, code, options) }
350
+ end
351
+
352
+ # Mirror the Prism.parse_file_comments API by using the serialization
353
+ # API. This uses native strings instead of Ruby strings because it allows us
354
+ # to use mmap when it is available.
355
+ def parse_file_comments(filepath, **options)
356
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_comments_common(string, string.read, options) }
357
+ end
358
+
359
+ # Mirror the Prism.parse_lex API by using the serialization API.
360
+ def parse_lex(code, **options)
361
+ LibRubyParser::PrismSource.with_string(code) { |string| parse_lex_common(string, code, options) }
362
+ end
363
+
364
+ # Mirror the Prism.parse_lex_file API by using the serialization API.
365
+ def parse_lex_file(filepath, **options)
366
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_lex_common(string, string.read, options) }
367
+ end
368
+
369
+ # Mirror the Prism.parse_success? API by using the serialization API.
370
+ def parse_success?(code, **options)
371
+ LibRubyParser::PrismSource.with_string(code) { |string| parse_file_success_common(string, options) }
372
+ end
373
+
374
+ # Mirror the Prism.parse_failure? API by using the serialization API.
375
+ def parse_failure?(code, **options)
376
+ !parse_success?(code, **options)
377
+ end
378
+
379
+ # Mirror the Prism.parse_file_success? API by using the serialization API.
380
+ def parse_file_success?(filepath, **options)
381
+ LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_file_success_common(string, options) }
382
+ end
383
+
384
+ # Mirror the Prism.parse_file_failure? API by using the serialization API.
385
+ def parse_file_failure?(filepath, **options)
386
+ !parse_file_success?(filepath, **options)
387
+ end
388
+
389
+ # Mirror the Prism.profile API by using the serialization API.
390
+ def profile(source, **options)
391
+ LibRubyParser::PrismSource.with_string(source) do |string|
392
+ if (format_type = raise_error_format_type(options))
393
+ raise_error(string, options, format_type)
394
+ end
395
+
396
+ LibRubyParser::PrismBuffer.with do |buffer|
397
+ LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
398
+ nil
399
+ end
400
+ end
401
+ end
402
+
403
+ # Mirror the Prism.profile_file API by using the serialization API.
404
+ def profile_file(filepath, **options)
405
+ LibRubyParser::PrismSource.with_file(filepath, options) do |string|
406
+ if (format_type = raise_error_format_type(options))
407
+ raise_error(string, options, format_type)
408
+ end
409
+
410
+ LibRubyParser::PrismBuffer.with do |buffer|
411
+ LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
412
+ nil
413
+ end
414
+ end
415
+ end
416
+
417
+ private
418
+
419
+ def dump_common(string, options) # :nodoc:
420
+ if (format_type = raise_error_format_type(options))
421
+ raise_error(string, options, format_type)
422
+ end
423
+
424
+ LibRubyParser::PrismBuffer.with do |buffer|
425
+ LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
426
+
427
+ dumped = buffer.read
428
+ dumped.freeze if options.fetch(:freeze, false)
429
+
430
+ dumped
431
+ end
432
+ end
433
+
434
+ def lex_common(string, code, options) # :nodoc:
435
+ format_type = raise_error_format_type(options)
436
+
437
+ LibRubyParser::PrismBuffer.with do |buffer|
438
+ LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
439
+ result = Serialize.load_lex(code, buffer.read, options.fetch(:freeze, false))
440
+
441
+ raise_error(string, options, format_type) if format_type && result.failure?
442
+ result
443
+ end
444
+ end
445
+
446
+ def parse_common(string, code, options) # :nodoc:
447
+ format_type = raise_error_format_type(options)
448
+ serialized = dump_common(string, options)
449
+ result = Serialize.load_parse(code, serialized, options.fetch(:freeze, false))
450
+
451
+ raise_error(string, options, format_type) if format_type && result.failure?
452
+ result
453
+ end
454
+
455
+ def parse_comments_common(string, code, options) # :nodoc:
456
+ if (format_type = raise_error_format_type(options))
457
+ raise_error(string, options, format_type)
458
+ end
459
+
460
+ LibRubyParser::PrismBuffer.with do |buffer|
461
+ LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options))
462
+ Serialize.load_parse_comments(code, buffer.read, options.fetch(:freeze, false))
463
+ end
464
+ end
465
+
466
+ def parse_lex_common(string, code, options) # :nodoc:
467
+ format_type = raise_error_format_type(options)
468
+
469
+ LibRubyParser::PrismBuffer.with do |buffer|
470
+ LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
471
+ result = Serialize.load_parse_lex(code, buffer.read, options.fetch(:freeze, false))
472
+
473
+ raise_error(string, options, format_type) if format_type && result.failure?
474
+ result
475
+ end
476
+ end
477
+
478
+ def parse_file_success_common(string, options) # :nodoc:
479
+ format_type = raise_error_format_type(options)
480
+ success = LibRubyParser.pm_serialize_parse_success_p(string.pointer, string.length, dump_options(options))
481
+
482
+ raise_error(string, options, format_type) if format_type && !success
483
+ success
484
+ end
485
+
486
+ # Extract the raise_error option from the given options hash and convert
487
+ # it into the format type that should be used when formatting errors, or
488
+ # nil if raising is disabled.
489
+ def raise_error_format_type(options) # :nodoc:
490
+ case (value = options.delete(:raise_error))
491
+ when nil, false
492
+ nil
493
+ when true
494
+ # When given true, mirror the behavior of CRuby itself: color when
495
+ # $stderr is a terminal (unless NO_COLOR is set), bold styling when
496
+ # NO_COLOR is set, plain otherwise. This policy is duplicated in
497
+ # ext/prism/extension.c (build_options_i) and the two must stay in
498
+ # sync.
499
+ if $stderr.respond_to?(:tty?) && $stderr.tty?
500
+ no_color = ENV["NO_COLOR"]
501
+ (no_color.nil? || no_color.empty?) ? :PM_ERRORS_FORMAT_COLOR : :PM_ERRORS_FORMAT_STYLE
502
+ else
503
+ :PM_ERRORS_FORMAT_PLAIN
504
+ end
505
+ when :plain
506
+ :PM_ERRORS_FORMAT_PLAIN
507
+ when :style
508
+ :PM_ERRORS_FORMAT_STYLE
509
+ when :color
510
+ :PM_ERRORS_FORMAT_COLOR
511
+ when Symbol
512
+ raise ArgumentError, "invalid raise_error value: #{value}"
513
+ else
514
+ raise TypeError, "wrong argument type #{value.class} (expected Symbol)"
515
+ end
516
+ end
517
+
518
+ # Parse the given source and format any errors that are encountered into
519
+ # an appropriate exception, then raise it. If the source parses without
520
+ # any errors, then return nil.
521
+ def raise_error(string, options, format_type) # :nodoc:
522
+ LibRubyParser::PrismBuffer.with do |buffer|
523
+ level = LibRubyParser.pm_serialize_parse_errors_format(buffer.pointer, string.pointer, string.length, dump_options(options), format_type)
524
+ return if level == -1
525
+
526
+ encoding_name, _, message = buffer.read.partition("\0")
527
+
528
+ case level
529
+ when 0 # syntax
530
+ error = SyntaxError.new(message.force_encoding(encoding_name))
531
+ error.instance_variable_set(:@path, options.fetch(:filepath, ""))
532
+ raise error
533
+ when 1 # argument
534
+ raise ArgumentError, message.force_encoding(encoding_name)
535
+ when 2 # load
536
+ error = LoadError.new(message.force_encoding(Encoding.find("locale")))
537
+ error.instance_variable_set(:@path, nil)
538
+ raise error
539
+ else
540
+ raise "Unknown error level: #{level}"
541
+ end
542
+ end
543
+ end
544
+
545
+ # Return the value that should be dumped for the command_line option.
546
+ def dump_options_command_line(options)
547
+ command_line = options.fetch(:command_line, "")
548
+ raise ArgumentError, "command_line must be a string" unless command_line.is_a?(String)
549
+
550
+ command_line.each_char.inject(0) do |value, char|
551
+ case char
552
+ when "a" then value | 0b000001
553
+ when "e" then value | 0b000010
554
+ when "l" then value | 0b000100
555
+ when "n" then value | 0b001000
556
+ when "p" then value | 0b010000
557
+ when "x" then value | 0b100000
558
+ else raise ArgumentError, "invalid command_line option: #{char}"
559
+ end
560
+ end
561
+ end
562
+
563
+ # Return the value that should be dumped for the version option.
564
+ def dump_options_version(version)
565
+ case version
566
+ when "current"
567
+ version_string_to_number(RUBY_VERSION) || raise(CurrentVersionError, RUBY_VERSION)
568
+ when "latest"
569
+ 0 # Handled in pm_parser_init
570
+ when "nearest", nil
571
+ dump = version_string_to_number(RUBY_VERSION)
572
+ return dump if dump
573
+ if RUBY_VERSION < "3.3"
574
+ version_string_to_number("3.3")
575
+ else
576
+ 0 # Handled in pm_parser_init
577
+ end
578
+ else
579
+ version_string_to_number(version) || raise(ArgumentError, "invalid version: #{version}")
580
+ end
581
+ end
582
+
583
+ # Converts a version string like "4.0.0" or "4.0" into a number.
584
+ # Returns nil if the version is unknown.
585
+ def version_string_to_number(version)
586
+ case version
587
+ when /\A3\.3(\.\d+)?\z/
588
+ 1
589
+ when /\A3\.4(\.\d+)?\z/
590
+ 2
591
+ when /\A3\.5(\.\d+)?\z/, /\A4\.0(\.\d+)?\z/
592
+ 3
593
+ when /\A4\.1(\.\d+)?\z/
594
+ 4
595
+ end
596
+ end
597
+
598
+ # The set of options that are understood by the parsing APIs. Note that
599
+ # raise_error is not listed here because it is deleted from the options
600
+ # hash by raise_error_format_type before the options are dumped.
601
+ DUMP_OPTIONS_KEYS = [:attach_comments, :command_line, :encoding, :filepath, :freeze, :frozen_string_literal, :line, :main_script, :partial_script, :scopes, :version].freeze
602
+ private_constant :DUMP_OPTIONS_KEYS
603
+
604
+ # Convert the given options into a serialized options string.
605
+ def dump_options(options)
606
+ unknown_keys = options.keys - DUMP_OPTIONS_KEYS
607
+ raise ArgumentError, "unknown keyword: #{unknown_keys.first}" unless unknown_keys.empty?
608
+
609
+ template = +""
610
+ values = []
611
+
612
+ template << "L"
613
+ if (filepath = options[:filepath])
614
+ values.push(filepath.bytesize, filepath.b)
615
+ template << "A*"
616
+ else
617
+ values << 0
618
+ end
619
+
620
+ template << "l"
621
+ values << options.fetch(:line, 1)
622
+
623
+ template << "L"
624
+ if (encoding = options[:encoding])
625
+ name = encoding.is_a?(Encoding) ? encoding.name : encoding
626
+ values.push(name.bytesize, name.b)
627
+ template << "A*"
628
+ else
629
+ values << 0
630
+ end
631
+
632
+ template << "C"
633
+ values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
634
+
635
+ template << "C"
636
+ values << dump_options_command_line(options)
637
+
638
+ template << "C"
639
+ values << dump_options_version(options[:version])
640
+
641
+ template << "C"
642
+ values << (options[:encoding] == false ? 1 : 0)
643
+
644
+ template << "C"
645
+ values << (options.fetch(:main_script, false) ? 1 : 0)
646
+
647
+ template << "C"
648
+ values << (options.fetch(:partial_script, false) ? 1 : 0)
649
+
650
+ template << "C"
651
+ values << (options.fetch(:freeze, false) ? 1 : 0)
652
+
653
+ template << "C"
654
+ values << (options.fetch(:attach_comments, false) ? 1 : 0)
655
+
656
+ template << "L"
657
+ if (scopes = options[:scopes])
658
+ values << scopes.length
659
+
660
+ scopes.each do |scope|
661
+ locals = nil
662
+ forwarding = 0
663
+
664
+ case scope
665
+ when Array
666
+ locals = scope
667
+ when Scope
668
+ locals = scope.locals
669
+
670
+ scope.forwarding.each do |forward|
671
+ case forward
672
+ when :* then forwarding |= 0x1
673
+ when :** then forwarding |= 0x2
674
+ when :& then forwarding |= 0x4
675
+ when :"..." then forwarding |= 0x8
676
+ else raise ArgumentError, "invalid forwarding value: #{forward}"
677
+ end
678
+ end
679
+ else
680
+ raise TypeError, "wrong argument type #{scope.class.inspect} (expected Array or Prism::Scope)"
681
+ end
682
+
683
+ template << "L"
684
+ values << locals.length
685
+
686
+ template << "C"
687
+ values << forwarding
688
+
689
+ locals.each do |local|
690
+ name = local.name
691
+ template << "L"
692
+ values << name.bytesize
693
+
694
+ template << "A*"
695
+ values << name.b
696
+ end
697
+ end
698
+ else
699
+ values << 0
700
+ end
701
+
702
+ values.pack(template)
703
+ end
704
+ end
705
+
706
+ # Here we are going to patch StringQuery to put in the class-level methods so
707
+ # that it can maintain a consistent interface
708
+ class StringQuery # :nodoc:
709
+ class << self
710
+ # Mirrors the C extension's StringQuery::local? method.
711
+ def local?(string)
712
+ query(LibRubyParser.pm_string_query_local(string, string.bytesize, string.encoding.name))
713
+ end
714
+
715
+ # Mirrors the C extension's StringQuery::constant? method.
716
+ def constant?(string)
717
+ query(LibRubyParser.pm_string_query_constant(string, string.bytesize, string.encoding.name))
718
+ end
719
+
720
+ # Mirrors the C extension's StringQuery::method_name? method.
721
+ def method_name?(string)
722
+ query(LibRubyParser.pm_string_query_method_name(string, string.bytesize, string.encoding.name))
723
+ end
724
+
725
+ private
726
+
727
+ # Parse the enum result and return an appropriate boolean.
728
+ def query(result)
729
+ case result
730
+ when :PM_STRING_QUERY_ERROR
731
+ raise ArgumentError, "Invalid or non ascii-compatible encoding"
732
+ when :PM_STRING_QUERY_FALSE
733
+ false
734
+ when :PM_STRING_QUERY_TRUE
735
+ true
736
+ end
737
+ end
738
+ end
739
+ end
740
+ end