prism 0.16.0 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +23 -1
  3. data/Makefile +6 -0
  4. data/README.md +1 -1
  5. data/config.yml +50 -35
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/releasing.md +4 -1
  8. data/docs/serialization.md +28 -29
  9. data/ext/prism/api_node.c +802 -770
  10. data/ext/prism/api_pack.c +20 -9
  11. data/ext/prism/extension.c +465 -160
  12. data/ext/prism/extension.h +1 -1
  13. data/include/prism/ast.h +3173 -763
  14. data/include/prism/defines.h +32 -9
  15. data/include/prism/diagnostic.h +36 -3
  16. data/include/prism/enc/pm_encoding.h +118 -28
  17. data/include/prism/node.h +38 -13
  18. data/include/prism/options.h +204 -0
  19. data/include/prism/pack.h +44 -33
  20. data/include/prism/parser.h +445 -200
  21. data/include/prism/prettyprint.h +12 -1
  22. data/include/prism/regexp.h +16 -2
  23. data/include/prism/util/pm_buffer.h +94 -16
  24. data/include/prism/util/pm_char.h +162 -48
  25. data/include/prism/util/pm_constant_pool.h +126 -32
  26. data/include/prism/util/pm_list.h +68 -38
  27. data/include/prism/util/pm_memchr.h +18 -3
  28. data/include/prism/util/pm_newline_list.h +70 -27
  29. data/include/prism/util/pm_state_stack.h +25 -7
  30. data/include/prism/util/pm_string.h +115 -27
  31. data/include/prism/util/pm_string_list.h +25 -6
  32. data/include/prism/util/pm_strncasecmp.h +32 -0
  33. data/include/prism/util/pm_strpbrk.h +31 -17
  34. data/include/prism/version.h +28 -3
  35. data/include/prism.h +224 -31
  36. data/lib/prism/compiler.rb +6 -3
  37. data/lib/prism/debug.rb +23 -7
  38. data/lib/prism/dispatcher.rb +33 -18
  39. data/lib/prism/dsl.rb +10 -5
  40. data/lib/prism/ffi.rb +132 -80
  41. data/lib/prism/lex_compat.rb +25 -15
  42. data/lib/prism/mutation_compiler.rb +10 -5
  43. data/lib/prism/node.rb +370 -135
  44. data/lib/prism/node_ext.rb +1 -1
  45. data/lib/prism/node_inspector.rb +1 -1
  46. data/lib/prism/pack.rb +79 -40
  47. data/lib/prism/parse_result/comments.rb +7 -2
  48. data/lib/prism/parse_result/newlines.rb +4 -0
  49. data/lib/prism/parse_result.rb +150 -30
  50. data/lib/prism/pattern.rb +11 -0
  51. data/lib/prism/ripper_compat.rb +28 -10
  52. data/lib/prism/serialize.rb +87 -55
  53. data/lib/prism/visitor.rb +10 -3
  54. data/lib/prism.rb +20 -2
  55. data/prism.gemspec +4 -2
  56. data/rbi/prism.rbi +5545 -5505
  57. data/rbi/prism_static.rbi +141 -131
  58. data/sig/prism.rbs +72 -43
  59. data/sig/prism_static.rbs +14 -1
  60. data/src/diagnostic.c +56 -53
  61. data/src/enc/pm_big5.c +1 -0
  62. data/src/enc/pm_euc_jp.c +1 -0
  63. data/src/enc/pm_gbk.c +1 -0
  64. data/src/enc/pm_shift_jis.c +1 -0
  65. data/src/enc/pm_tables.c +316 -80
  66. data/src/enc/pm_unicode.c +53 -8
  67. data/src/enc/pm_windows_31j.c +1 -0
  68. data/src/node.c +334 -321
  69. data/src/options.c +170 -0
  70. data/src/prettyprint.c +74 -47
  71. data/src/prism.c +1642 -856
  72. data/src/regexp.c +151 -95
  73. data/src/serialize.c +44 -20
  74. data/src/token_type.c +3 -1
  75. data/src/util/pm_buffer.c +45 -15
  76. data/src/util/pm_char.c +103 -57
  77. data/src/util/pm_constant_pool.c +51 -21
  78. data/src/util/pm_list.c +12 -4
  79. data/src/util/pm_memchr.c +5 -3
  80. data/src/util/pm_newline_list.c +20 -12
  81. data/src/util/pm_state_stack.c +9 -3
  82. data/src/util/pm_string.c +95 -85
  83. data/src/util/pm_string_list.c +14 -15
  84. data/src/util/pm_strncasecmp.c +10 -3
  85. data/src/util/pm_strpbrk.c +25 -19
  86. metadata +5 -3
  87. data/docs/prism.png +0 -0
data/lib/prism/debug.rb CHANGED
@@ -4,7 +4,9 @@ module Prism
4
4
  # This module is used for testing and debugging and is not meant to be used by
5
5
  # consumers of this library.
6
6
  module Debug
7
- class ISeq
7
+ # A wrapper around a RubyVM::InstructionSequence that provides a more
8
+ # convenient interface for accessing parts of the iseq.
9
+ class ISeq # :nodoc:
8
10
  attr_reader :parts
9
11
 
10
12
  def initialize(parts)
@@ -42,6 +44,11 @@ module Prism
42
44
  end
43
45
  end
44
46
 
47
+ private_constant :ISeq
48
+
49
+ # :call-seq:
50
+ # Debug::cruby_locals(source) -> Array
51
+ #
45
52
  # For the given source, compiles with CRuby and returns a list of all of the
46
53
  # sets of local variables that were encountered.
47
54
  def self.cruby_locals(source)
@@ -76,8 +83,16 @@ module Prism
76
83
  end
77
84
  end
78
85
 
86
+ # Used to hold the place of a local that will be in the local table but
87
+ # cannot be accessed directly from the source code. For example, the
88
+ # iteration variable in a for loop or the positional parameter on a method
89
+ # definition that is destructured.
79
90
  AnonymousLocal = Object.new
91
+ private_constant :AnonymousLocal
80
92
 
93
+ # :call-seq:
94
+ # Debug::prism_locals(source) -> Array
95
+ #
81
96
  # For the given source, parses with prism and returns a list of all of the
82
97
  # sets of local variables that were encountered.
83
98
  def self.prism_locals(source)
@@ -114,8 +129,8 @@ module Prism
114
129
  AnonymousLocal
115
130
  end
116
131
  end,
117
- *params.keywords.reject(&:value).map(&:name),
118
- *params.keywords.select(&:value).map(&:name)
132
+ *params.keywords.grep(RequiredKeywordParameterNode).map(&:name),
133
+ *params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
119
134
  ]
120
135
 
121
136
  sorted << AnonymousLocal if params.keywords.any?
@@ -164,12 +179,13 @@ module Prism
164
179
  locals
165
180
  end
166
181
 
182
+ # :call-seq:
183
+ # Debug::newlines(source) -> Array
184
+ #
185
+ # For the given source string, return the byte offsets of every newline in
186
+ # the source.
167
187
  def self.newlines(source)
168
188
  Prism.parse(source).source.offsets
169
189
  end
170
-
171
- def self.parse_serialize_file(filepath)
172
- parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L"))
173
- end
174
190
  end
175
191
  end
@@ -41,23 +41,24 @@ module Prism
41
41
  # attr_reader listeners: Hash[Symbol, Array[Listener]]
42
42
  attr_reader :listeners
43
43
 
44
+ # Initialize a new dispatcher.
44
45
  def initialize
45
46
  @listeners = {}
46
47
  end
47
48
 
48
- # Register a listener for one or more events
49
+ # Register a listener for one or more events.
49
50
  #
50
51
  # def register: (Listener, *Symbol) -> void
51
52
  def register(listener, *events)
52
53
  events.each { |event| (listeners[event] ||= []) << listener }
53
54
  end
54
55
 
55
- # Walks `root` dispatching events to all registered listeners
56
+ # Walks `root` dispatching events to all registered listeners.
56
57
  #
57
58
  # def dispatch: (Node) -> void
58
59
  alias dispatch visit
59
60
 
60
- # Dispatches a single event for `node` to all registered listeners
61
+ # Dispatches a single event for `node` to all registered listeners.
61
62
  #
62
63
  # def dispatch_once: (Node) -> void
63
64
  def dispatch_once(node)
@@ -736,14 +737,6 @@ module Prism
736
737
  listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
737
738
  end
738
739
 
739
- # Dispatch enter and leave events for KeywordParameterNode nodes and continue
740
- # walking the tree.
741
- def visit_keyword_parameter_node(node)
742
- listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
743
- super
744
- listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
745
- end
746
-
747
740
  # Dispatch enter and leave events for KeywordRestParameterNode nodes and continue
748
741
  # walking the tree.
749
742
  def visit_keyword_rest_parameter_node(node)
@@ -904,6 +897,14 @@ module Prism
904
897
  listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
905
898
  end
906
899
 
900
+ # Dispatch enter and leave events for OptionalKeywordParameterNode nodes and continue
901
+ # walking the tree.
902
+ def visit_optional_keyword_parameter_node(node)
903
+ listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
904
+ super
905
+ listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
906
+ end
907
+
907
908
  # Dispatch enter and leave events for OptionalParameterNode nodes and continue
908
909
  # walking the tree.
909
910
  def visit_optional_parameter_node(node)
@@ -1008,6 +1009,14 @@ module Prism
1008
1009
  listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
1009
1010
  end
1010
1011
 
1012
+ # Dispatch enter and leave events for RequiredKeywordParameterNode nodes and continue
1013
+ # walking the tree.
1014
+ def visit_required_keyword_parameter_node(node)
1015
+ listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
1016
+ super
1017
+ listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
1018
+ end
1019
+
1011
1020
  # Dispatch enter and leave events for RequiredParameterNode nodes and continue
1012
1021
  # walking the tree.
1013
1022
  def visit_required_parameter_node(node)
@@ -1208,7 +1217,7 @@ module Prism
1208
1217
  listeners[:on_yield_node_leave]&.each { |listener| listener.on_yield_node_leave(node) }
1209
1218
  end
1210
1219
 
1211
- class DispatchOnce < Visitor
1220
+ class DispatchOnce < Visitor # :nodoc:
1212
1221
  attr_reader :listeners
1213
1222
 
1214
1223
  def initialize(listeners)
@@ -1719,12 +1728,6 @@ module Prism
1719
1728
  listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
1720
1729
  end
1721
1730
 
1722
- # Dispatch enter and leave events for KeywordParameterNode nodes.
1723
- def visit_keyword_parameter_node(node)
1724
- listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
1725
- listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
1726
- end
1727
-
1728
1731
  # Dispatch enter and leave events for KeywordRestParameterNode nodes.
1729
1732
  def visit_keyword_rest_parameter_node(node)
1730
1733
  listeners[:on_keyword_rest_parameter_node_enter]&.each { |listener| listener.on_keyword_rest_parameter_node_enter(node) }
@@ -1845,6 +1848,12 @@ module Prism
1845
1848
  listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
1846
1849
  end
1847
1850
 
1851
+ # Dispatch enter and leave events for OptionalKeywordParameterNode nodes.
1852
+ def visit_optional_keyword_parameter_node(node)
1853
+ listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
1854
+ listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
1855
+ end
1856
+
1848
1857
  # Dispatch enter and leave events for OptionalParameterNode nodes.
1849
1858
  def visit_optional_parameter_node(node)
1850
1859
  listeners[:on_optional_parameter_node_enter]&.each { |listener| listener.on_optional_parameter_node_enter(node) }
@@ -1923,6 +1932,12 @@ module Prism
1923
1932
  listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
1924
1933
  end
1925
1934
 
1935
+ # Dispatch enter and leave events for RequiredKeywordParameterNode nodes.
1936
+ def visit_required_keyword_parameter_node(node)
1937
+ listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
1938
+ listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
1939
+ end
1940
+
1926
1941
  # Dispatch enter and leave events for RequiredParameterNode nodes.
1927
1942
  def visit_required_parameter_node(node)
1928
1943
  listeners[:on_required_parameter_node_enter]&.each { |listener| listener.on_required_parameter_node_enter(node) }
data/lib/prism/dsl.rb CHANGED
@@ -462,11 +462,6 @@ module Prism
462
462
  KeywordHashNode.new(elements, location)
463
463
  end
464
464
 
465
- # Create a new KeywordParameterNode node
466
- def KeywordParameterNode(name, name_loc, value, location = Location())
467
- KeywordParameterNode.new(name, name_loc, value, location)
468
- end
469
-
470
465
  # Create a new KeywordRestParameterNode node
471
466
  def KeywordRestParameterNode(name, name_loc, operator_loc, location = Location())
472
467
  KeywordRestParameterNode.new(name, name_loc, operator_loc, location)
@@ -567,6 +562,11 @@ module Prism
567
562
  NumberedReferenceReadNode.new(number, location)
568
563
  end
569
564
 
565
+ # Create a new OptionalKeywordParameterNode node
566
+ def OptionalKeywordParameterNode(name, name_loc, value, location = Location())
567
+ OptionalKeywordParameterNode.new(name, name_loc, value, location)
568
+ end
569
+
570
570
  # Create a new OptionalParameterNode node
571
571
  def OptionalParameterNode(name, name_loc, operator_loc, value, location = Location())
572
572
  OptionalParameterNode.new(name, name_loc, operator_loc, value, location)
@@ -632,6 +632,11 @@ module Prism
632
632
  RegularExpressionNode.new(opening_loc, content_loc, closing_loc, unescaped, flags, location)
633
633
  end
634
634
 
635
+ # Create a new RequiredKeywordParameterNode node
636
+ def RequiredKeywordParameterNode(name, name_loc, location = Location())
637
+ RequiredKeywordParameterNode.new(name, name_loc, location)
638
+ end
639
+
635
640
  # Create a new RequiredParameterNode node
636
641
  def RequiredParameterNode(name, location = Location())
637
642
  RequiredParameterNode.new(name, location)
data/lib/prism/ffi.rb CHANGED
@@ -9,7 +9,7 @@ require "ffi"
9
9
  module Prism
10
10
  BACKEND = :FFI
11
11
 
12
- module LibRubyParser
12
+ module LibRubyParser # :nodoc:
13
13
  extend FFI::Library
14
14
 
15
15
  # Define the library that we will be pulling functions from. Note that this
@@ -69,10 +69,10 @@ module Prism
69
69
  load_exported_functions_from(
70
70
  "prism.h",
71
71
  "pm_version",
72
- "pm_parse_serialize",
73
- "pm_parse_serialize_comments",
74
- "pm_lex_serialize",
75
- "pm_parse_lex_serialize"
72
+ "pm_serialize_parse",
73
+ "pm_serialize_parse_comments",
74
+ "pm_serialize_lex",
75
+ "pm_serialize_parse_lex"
76
76
  )
77
77
 
78
78
  load_exported_functions_from(
@@ -95,7 +95,7 @@ module Prism
95
95
 
96
96
  # This object represents a pm_buffer_t. We only use it as an opaque pointer,
97
97
  # so it doesn't need to know the fields of pm_buffer_t.
98
- class PrismBuffer
98
+ class PrismBuffer # :nodoc:
99
99
  SIZEOF = LibRubyParser.pm_buffer_sizeof
100
100
 
101
101
  attr_reader :pointer
@@ -133,7 +133,7 @@ module Prism
133
133
 
134
134
  # This object represents a pm_string_t. We only use it as an opaque pointer,
135
135
  # so it doesn't have to be an FFI::Struct.
136
- class PrismString
136
+ class PrismString # :nodoc:
137
137
  SIZEOF = LibRubyParser.pm_string_sizeof
138
138
 
139
139
  attr_reader :pointer
@@ -167,14 +167,6 @@ module Prism
167
167
  end
168
168
  end
169
169
  end
170
-
171
- def self.dump_internal(source, source_size, filepath)
172
- PrismBuffer.with do |buffer|
173
- metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
174
- pm_parse_serialize(source, source_size, buffer.pointer, metadata)
175
- buffer.read
176
- end
177
- end
178
170
  end
179
171
 
180
172
  # Mark the LibRubyParser module as private as it should only be called through
@@ -184,93 +176,153 @@ module Prism
184
176
  # The version constant is set by reading the result of calling pm_version.
185
177
  VERSION = LibRubyParser.pm_version.read_string
186
178
 
187
- # Mirror the Prism.dump API by using the serialization API.
188
- def self.dump(code, filepath = nil)
189
- LibRubyParser.dump_internal(code, code.bytesize, filepath)
190
- end
179
+ class << self
180
+ # Mirror the Prism.dump API by using the serialization API.
181
+ def dump(code, **options)
182
+ LibRubyParser::PrismBuffer.with do |buffer|
183
+ LibRubyParser.pm_serialize_parse(buffer.pointer, code, code.bytesize, dump_options(options))
184
+ buffer.read
185
+ end
186
+ end
191
187
 
192
- # Mirror the Prism.dump_file API by using the serialization API.
193
- def self.dump_file(filepath)
194
- LibRubyParser::PrismString.with(filepath) do |string|
195
- LibRubyParser.dump_internal(string.source, string.length, filepath)
188
+ # Mirror the Prism.dump_file API by using the serialization API.
189
+ def dump_file(filepath, **options)
190
+ LibRubyParser::PrismString.with(filepath) do |string|
191
+ dump(string.read, **options, filepath: filepath)
192
+ end
196
193
  end
197
- end
198
194
 
199
- # Mirror the Prism.lex API by using the serialization API.
200
- def self.lex(code, filepath = nil)
201
- LibRubyParser::PrismBuffer.with do |buffer|
202
- LibRubyParser.pm_lex_serialize(code, code.bytesize, filepath, buffer.pointer)
203
- Serialize.load_tokens(Source.new(code), buffer.read)
195
+ # Mirror the Prism.lex API by using the serialization API.
196
+ def lex(code, **options)
197
+ LibRubyParser::PrismBuffer.with do |buffer|
198
+ LibRubyParser.pm_serialize_lex(buffer.pointer, code, code.bytesize, dump_options(options))
199
+ Serialize.load_tokens(Source.new(code), buffer.read)
200
+ end
204
201
  end
205
- end
206
202
 
207
- # Mirror the Prism.lex_file API by using the serialization API.
208
- def self.lex_file(filepath)
209
- LibRubyParser::PrismString.with(filepath) do |string|
210
- lex(string.read, filepath)
203
+ # Mirror the Prism.lex_file API by using the serialization API.
204
+ def lex_file(filepath, **options)
205
+ LibRubyParser::PrismString.with(filepath) do |string|
206
+ lex(string.read, **options, filepath: filepath)
207
+ end
211
208
  end
212
- end
213
209
 
214
- # Mirror the Prism.parse API by using the serialization API.
215
- def self.parse(code, filepath = nil)
216
- Prism.load(code, dump(code, filepath))
217
- end
210
+ # Mirror the Prism.parse API by using the serialization API.
211
+ def parse(code, **options)
212
+ Prism.load(code, dump(code, **options))
213
+ end
218
214
 
219
- # Mirror the Prism.parse_file API by using the serialization API. This uses
220
- # native strings instead of Ruby strings because it allows us to use mmap when
221
- # it is available.
222
- def self.parse_file(filepath)
223
- LibRubyParser::PrismString.with(filepath) do |string|
224
- parse(string.read, filepath)
215
+ # Mirror the Prism.parse_file API by using the serialization API. This uses
216
+ # native strings instead of Ruby strings because it allows us to use mmap when
217
+ # it is available.
218
+ def parse_file(filepath, **options)
219
+ LibRubyParser::PrismString.with(filepath) do |string|
220
+ parse(string.read, **options, filepath: filepath)
221
+ end
225
222
  end
226
- end
227
223
 
228
- # Mirror the Prism.parse_comments API by using the serialization API.
229
- def self.parse_comments(code, filepath = nil)
230
- LibRubyParser::PrismBuffer.with do |buffer|
231
- metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
232
- LibRubyParser.pm_parse_serialize_comments(code, code.bytesize, buffer.pointer, metadata)
224
+ # Mirror the Prism.parse_comments API by using the serialization API.
225
+ def parse_comments(code, **options)
226
+ LibRubyParser::PrismBuffer.with do |buffer|
227
+ LibRubyParser.pm_serialize_parse_comments(buffer.pointer, code, code.bytesize, dump_options(options))
233
228
 
234
- source = Source.new(code)
235
- loader = Serialize::Loader.new(source, buffer.read)
229
+ source = Source.new(code)
230
+ loader = Serialize::Loader.new(source, buffer.read)
236
231
 
237
- loader.load_header
238
- loader.load_force_encoding
239
- loader.load_comments
232
+ loader.load_header
233
+ loader.load_force_encoding
234
+ loader.load_start_line
235
+ loader.load_comments
236
+ end
240
237
  end
241
- end
242
238
 
243
- # Mirror the Prism.parse_file_comments API by using the serialization
244
- # API. This uses native strings instead of Ruby strings because it allows us
245
- # to use mmap when it is available.
246
- def self.parse_file_comments(filepath)
247
- LibRubyParser::PrismString.with(filepath) do |string|
248
- parse_comments(string.read, filepath)
239
+ # Mirror the Prism.parse_file_comments API by using the serialization
240
+ # API. This uses native strings instead of Ruby strings because it allows us
241
+ # to use mmap when it is available.
242
+ def parse_file_comments(filepath, **options)
243
+ LibRubyParser::PrismString.with(filepath) do |string|
244
+ parse_comments(string.read, **options, filepath: filepath)
245
+ end
249
246
  end
250
- end
251
247
 
252
- # Mirror the Prism.parse_lex API by using the serialization API.
253
- def self.parse_lex(code, filepath = nil)
254
- LibRubyParser::PrismBuffer.with do |buffer|
255
- metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
256
- LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata)
248
+ # Mirror the Prism.parse_lex API by using the serialization API.
249
+ def parse_lex(code, **options)
250
+ LibRubyParser::PrismBuffer.with do |buffer|
251
+ LibRubyParser.pm_serialize_parse_lex(buffer.pointer, code, code.bytesize, dump_options(options))
257
252
 
258
- source = Source.new(code)
259
- loader = Serialize::Loader.new(source, buffer.read)
253
+ source = Source.new(code)
254
+ loader = Serialize::Loader.new(source, buffer.read)
260
255
 
261
- tokens = loader.load_tokens
262
- node, comments, magic_comments, errors, warnings = loader.load_nodes
256
+ tokens = loader.load_tokens
257
+ node, comments, magic_comments, errors, warnings = loader.load_nodes
258
+ tokens.each { |token,| token.value.force_encoding(loader.encoding) }
263
259
 
264
- tokens.each { |token,| token.value.force_encoding(loader.encoding) }
260
+ ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
261
+ end
262
+ end
265
263
 
266
- ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
264
+ # Mirror the Prism.parse_lex_file API by using the serialization API.
265
+ def parse_lex_file(filepath, **options)
266
+ LibRubyParser::PrismString.with(filepath) do |string|
267
+ parse_lex(string.read, **options, filepath: filepath)
268
+ end
267
269
  end
268
- end
269
270
 
270
- # Mirror the Prism.parse_lex_file API by using the serialization API.
271
- def self.parse_lex_file(filepath)
272
- LibRubyParser::PrismString.with(filepath) do |string|
273
- parse_lex(string.read, filepath)
271
+ private
272
+
273
+ # Convert the given options into a serialized options string.
274
+ def dump_options(options)
275
+ template = +""
276
+ values = []
277
+
278
+ template << "L"
279
+ if (filepath = options[:filepath])
280
+ values.push(filepath.bytesize, filepath.b)
281
+ template << "A*"
282
+ else
283
+ values << 0
284
+ end
285
+
286
+ template << "L"
287
+ values << options.fetch(:line, 1)
288
+
289
+ template << "L"
290
+ if (encoding = options[:encoding])
291
+ name = encoding.name
292
+ values.push(name.bytesize, name.b)
293
+ template << "A*"
294
+ else
295
+ values << 0
296
+ end
297
+
298
+ template << "C"
299
+ values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
300
+
301
+ template << "C"
302
+ values << (options[:verbose] ? 0 : 1)
303
+
304
+ template << "L"
305
+ if (scopes = options[:scopes])
306
+ values << scopes.length
307
+
308
+ scopes.each do |scope|
309
+ template << "L"
310
+ values << scope.length
311
+
312
+ scope.each do |local|
313
+ name = local.name
314
+ template << "L"
315
+ values << name.bytesize
316
+
317
+ template << "A*"
318
+ values << name.b
319
+ end
320
+ end
321
+ else
322
+ values << 0
323
+ end
324
+
325
+ values.pack(template)
274
326
  end
275
327
  end
276
328
  end
@@ -8,7 +8,7 @@ module Prism
8
8
  # of cases, this is a one-to-one mapping of the token type. Everything else
9
9
  # generally lines up. However, there are a few cases that require special
10
10
  # handling.
11
- class LexCompat
11
+ class LexCompat # :nodoc:
12
12
  # This is a mapping of prism token types to Ripper token types. This is a
13
13
  # many-to-one mapping because we split up our token types, whereas Ripper
14
14
  # tends to group them.
@@ -184,18 +184,22 @@ module Prism
184
184
  # However, we add a couple of convenience methods onto them to make them a
185
185
  # little easier to work with. We delegate all other methods to the array.
186
186
  class Token < SimpleDelegator
187
+ # The location of the token in the source.
187
188
  def location
188
189
  self[0]
189
190
  end
190
191
 
192
+ # The type of the token.
191
193
  def event
192
194
  self[1]
193
195
  end
194
196
 
197
+ # The slice of the source that this token represents.
195
198
  def value
196
199
  self[2]
197
200
  end
198
201
 
202
+ # The state of the lexer when this token was produced.
199
203
  def state
200
204
  self[3]
201
205
  end
@@ -204,7 +208,7 @@ module Prism
204
208
  # Ripper doesn't include the rest of the token in the event, so we need to
205
209
  # trim it down to just the content on the first line when comparing.
206
210
  class EndContentToken < Token
207
- def ==(other)
211
+ def ==(other) # :nodoc:
208
212
  [self[0], self[1], self[2][0..self[2].index("\n")], self[3]] == other
209
213
  end
210
214
  end
@@ -212,7 +216,7 @@ module Prism
212
216
  # Tokens where state should be ignored
213
217
  # used for :on_comment, :on_heredoc_end, :on_embexpr_end
214
218
  class IgnoreStateToken < Token
215
- def ==(other)
219
+ def ==(other) # :nodoc:
216
220
  self[0...-1] == other[0...-1]
217
221
  end
218
222
  end
@@ -222,7 +226,7 @@ module Prism
222
226
  # through named captures in regular expressions). In that case we don't
223
227
  # compare the state.
224
228
  class IdentToken < Token
225
- def ==(other)
229
+ def ==(other) # :nodoc:
226
230
  (self[0...-1] == other[0...-1]) && (
227
231
  (other[3] == Ripper::EXPR_LABEL | Ripper::EXPR_END) ||
228
232
  (other[3] & Ripper::EXPR_ARG_ANY != 0)
@@ -233,7 +237,7 @@ module Prism
233
237
  # Ignored newlines can occasionally have a LABEL state attached to them, so
234
238
  # we compare the state differently here.
235
239
  class IgnoredNewlineToken < Token
236
- def ==(other)
240
+ def ==(other) # :nodoc:
237
241
  return false unless self[0...-1] == other[0...-1]
238
242
 
239
243
  if self[4] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
@@ -253,7 +257,7 @@ module Prism
253
257
  # more accurately, so we need to allow comparing against both END and
254
258
  # END|LABEL.
255
259
  class ParamToken < Token
256
- def ==(other)
260
+ def ==(other) # :nodoc:
257
261
  (self[0...-1] == other[0...-1]) && (
258
262
  (other[3] == Ripper::EXPR_END) ||
259
263
  (other[3] == Ripper::EXPR_END | Ripper::EXPR_LABEL)
@@ -264,12 +268,12 @@ module Prism
264
268
  # A heredoc in this case is a list of tokens that belong to the body of the
265
269
  # heredoc that should be appended onto the list of tokens when the heredoc
266
270
  # closes.
267
- module Heredoc
271
+ module Heredoc # :nodoc:
268
272
  # Heredocs that are no dash or tilde heredocs are just a list of tokens.
269
273
  # We need to keep them around so that we can insert them in the correct
270
274
  # order back into the token stream and set the state of the last token to
271
275
  # the state that the heredoc was opened in.
272
- class PlainHeredoc
276
+ class PlainHeredoc # :nodoc:
273
277
  attr_reader :tokens
274
278
 
275
279
  def initialize
@@ -288,7 +292,7 @@ module Prism
288
292
  # Dash heredocs are a little more complicated. They are a list of tokens
289
293
  # that need to be split on "\\\n" to mimic Ripper's behavior. We also need
290
294
  # to keep track of the state that the heredoc was opened in.
291
- class DashHeredoc
295
+ class DashHeredoc # :nodoc:
292
296
  attr_reader :split, :tokens
293
297
 
294
298
  def initialize(split)
@@ -347,7 +351,7 @@ module Prism
347
351
  # insert them into the stream in the correct order. As such, we can do
348
352
  # some extra manipulation on the tokens to make them match Ripper's
349
353
  # output by mirroring the dedent logic that Ripper uses.
350
- class DedentingHeredoc
354
+ class DedentingHeredoc # :nodoc:
351
355
  TAB_WIDTH = 8
352
356
 
353
357
  attr_reader :tokens, :dedent_next, :dedent, :embexpr_balance
@@ -588,11 +592,13 @@ module Prism
588
592
  end
589
593
  end
590
594
 
591
- attr_reader :source, :filepath
595
+ private_constant :Heredoc
592
596
 
593
- def initialize(source, filepath = "")
597
+ attr_reader :source, :options
598
+
599
+ def initialize(source, **options)
594
600
  @source = source
595
- @filepath = filepath || ""
601
+ @options = options
596
602
  end
597
603
 
598
604
  def result
@@ -601,7 +607,7 @@ module Prism
601
607
  state = :default
602
608
  heredoc_stack = [[]]
603
609
 
604
- result = Prism.lex(source, @filepath)
610
+ result = Prism.lex(source, **options)
605
611
  result_value = result.value
606
612
  previous_state = nil
607
613
 
@@ -829,9 +835,11 @@ module Prism
829
835
  end
830
836
  end
831
837
 
838
+ private_constant :LexCompat
839
+
832
840
  # This is a class that wraps the Ripper lexer to produce almost exactly the
833
841
  # same tokens.
834
- class LexRipper
842
+ class LexRipper # :nodoc:
835
843
  attr_reader :source
836
844
 
837
845
  def initialize(source)
@@ -869,4 +877,6 @@ module Prism
869
877
  results
870
878
  end
871
879
  end
880
+
881
+ private_constant :LexRipper
872
882
  end
@@ -430,11 +430,6 @@ module Prism
430
430
  node.copy(elements: visit_all(node.elements))
431
431
  end
432
432
 
433
- # Copy a KeywordParameterNode node
434
- def visit_keyword_parameter_node(node)
435
- node.copy(value: visit(node.value))
436
- end
437
-
438
433
  # Copy a KeywordRestParameterNode node
439
434
  def visit_keyword_rest_parameter_node(node)
440
435
  node.copy
@@ -535,6 +530,11 @@ module Prism
535
530
  node.copy
536
531
  end
537
532
 
533
+ # Copy a OptionalKeywordParameterNode node
534
+ def visit_optional_keyword_parameter_node(node)
535
+ node.copy(value: visit(node.value))
536
+ end
537
+
538
538
  # Copy a OptionalParameterNode node
539
539
  def visit_optional_parameter_node(node)
540
540
  node.copy(value: visit(node.value))
@@ -600,6 +600,11 @@ module Prism
600
600
  node.copy
601
601
  end
602
602
 
603
+ # Copy a RequiredKeywordParameterNode node
604
+ def visit_required_keyword_parameter_node(node)
605
+ node.copy
606
+ end
607
+
603
608
  # Copy a RequiredParameterNode node
604
609
  def visit_required_parameter_node(node)
605
610
  node.copy