prism 0.16.0 → 0.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -1
  3. data/Makefile +6 -0
  4. data/README.md +1 -1
  5. data/config.yml +50 -35
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/serialization.md +28 -29
  8. data/ext/prism/api_node.c +802 -770
  9. data/ext/prism/api_pack.c +20 -9
  10. data/ext/prism/extension.c +464 -162
  11. data/ext/prism/extension.h +1 -1
  12. data/include/prism/ast.h +3173 -763
  13. data/include/prism/defines.h +32 -9
  14. data/include/prism/diagnostic.h +36 -3
  15. data/include/prism/enc/pm_encoding.h +118 -28
  16. data/include/prism/node.h +38 -13
  17. data/include/prism/options.h +204 -0
  18. data/include/prism/pack.h +44 -33
  19. data/include/prism/parser.h +445 -200
  20. data/include/prism/prettyprint.h +12 -1
  21. data/include/prism/regexp.h +16 -2
  22. data/include/prism/util/pm_buffer.h +94 -16
  23. data/include/prism/util/pm_char.h +162 -48
  24. data/include/prism/util/pm_constant_pool.h +126 -32
  25. data/include/prism/util/pm_list.h +68 -38
  26. data/include/prism/util/pm_memchr.h +18 -3
  27. data/include/prism/util/pm_newline_list.h +70 -27
  28. data/include/prism/util/pm_state_stack.h +25 -7
  29. data/include/prism/util/pm_string.h +115 -27
  30. data/include/prism/util/pm_string_list.h +25 -6
  31. data/include/prism/util/pm_strncasecmp.h +32 -0
  32. data/include/prism/util/pm_strpbrk.h +31 -17
  33. data/include/prism/version.h +27 -2
  34. data/include/prism.h +224 -31
  35. data/lib/prism/compiler.rb +6 -3
  36. data/lib/prism/debug.rb +23 -7
  37. data/lib/prism/dispatcher.rb +33 -18
  38. data/lib/prism/dsl.rb +10 -5
  39. data/lib/prism/ffi.rb +132 -80
  40. data/lib/prism/lex_compat.rb +25 -15
  41. data/lib/prism/mutation_compiler.rb +10 -5
  42. data/lib/prism/node.rb +370 -135
  43. data/lib/prism/node_ext.rb +1 -1
  44. data/lib/prism/node_inspector.rb +1 -1
  45. data/lib/prism/pack.rb +79 -40
  46. data/lib/prism/parse_result/comments.rb +7 -2
  47. data/lib/prism/parse_result/newlines.rb +4 -0
  48. data/lib/prism/parse_result.rb +150 -30
  49. data/lib/prism/pattern.rb +11 -0
  50. data/lib/prism/ripper_compat.rb +28 -10
  51. data/lib/prism/serialize.rb +86 -54
  52. data/lib/prism/visitor.rb +10 -3
  53. data/lib/prism.rb +20 -2
  54. data/prism.gemspec +4 -2
  55. data/rbi/prism.rbi +104 -60
  56. data/rbi/prism_static.rbi +16 -2
  57. data/sig/prism.rbs +72 -43
  58. data/sig/prism_static.rbs +14 -1
  59. data/src/diagnostic.c +56 -53
  60. data/src/enc/pm_big5.c +1 -0
  61. data/src/enc/pm_euc_jp.c +1 -0
  62. data/src/enc/pm_gbk.c +1 -0
  63. data/src/enc/pm_shift_jis.c +1 -0
  64. data/src/enc/pm_tables.c +316 -80
  65. data/src/enc/pm_unicode.c +53 -8
  66. data/src/enc/pm_windows_31j.c +1 -0
  67. data/src/node.c +334 -321
  68. data/src/options.c +170 -0
  69. data/src/prettyprint.c +74 -47
  70. data/src/prism.c +1642 -856
  71. data/src/regexp.c +151 -95
  72. data/src/serialize.c +44 -20
  73. data/src/token_type.c +3 -1
  74. data/src/util/pm_buffer.c +45 -15
  75. data/src/util/pm_char.c +103 -57
  76. data/src/util/pm_constant_pool.c +51 -21
  77. data/src/util/pm_list.c +12 -4
  78. data/src/util/pm_memchr.c +5 -3
  79. data/src/util/pm_newline_list.c +20 -12
  80. data/src/util/pm_state_stack.c +9 -3
  81. data/src/util/pm_string.c +95 -85
  82. data/src/util/pm_string_list.c +14 -15
  83. data/src/util/pm_strncasecmp.c +10 -3
  84. data/src/util/pm_strpbrk.c +25 -19
  85. metadata +5 -3
  86. data/docs/prism.png +0 -0
data/lib/prism/debug.rb CHANGED
@@ -4,7 +4,9 @@ module Prism
4
4
  # This module is used for testing and debugging and is not meant to be used by
5
5
  # consumers of this library.
6
6
  module Debug
7
- class ISeq
7
+ # A wrapper around a RubyVM::InstructionSequence that provides a more
8
+ # convenient interface for accessing parts of the iseq.
9
+ class ISeq # :nodoc:
8
10
  attr_reader :parts
9
11
 
10
12
  def initialize(parts)
@@ -42,6 +44,11 @@ module Prism
42
44
  end
43
45
  end
44
46
 
47
+ private_constant :ISeq
48
+
49
+ # :call-seq:
50
+ # Debug::cruby_locals(source) -> Array
51
+ #
45
52
  # For the given source, compiles with CRuby and returns a list of all of the
46
53
  # sets of local variables that were encountered.
47
54
  def self.cruby_locals(source)
@@ -76,8 +83,16 @@ module Prism
76
83
  end
77
84
  end
78
85
 
86
+ # Used to hold the place of a local that will be in the local table but
87
+ # cannot be accessed directly from the source code. For example, the
88
+ # iteration variable in a for loop or the positional parameter on a method
89
+ # definition that is destructured.
79
90
  AnonymousLocal = Object.new
91
+ private_constant :AnonymousLocal
80
92
 
93
+ # :call-seq:
94
+ # Debug::prism_locals(source) -> Array
95
+ #
81
96
  # For the given source, parses with prism and returns a list of all of the
82
97
  # sets of local variables that were encountered.
83
98
  def self.prism_locals(source)
@@ -114,8 +129,8 @@ module Prism
114
129
  AnonymousLocal
115
130
  end
116
131
  end,
117
- *params.keywords.reject(&:value).map(&:name),
118
- *params.keywords.select(&:value).map(&:name)
132
+ *params.keywords.grep(RequiredKeywordParameterNode).map(&:name),
133
+ *params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
119
134
  ]
120
135
 
121
136
  sorted << AnonymousLocal if params.keywords.any?
@@ -164,12 +179,13 @@ module Prism
164
179
  locals
165
180
  end
166
181
 
182
+ # :call-seq:
183
+ # Debug::newlines(source) -> Array
184
+ #
185
+ # For the given source string, return the byte offsets of every newline in
186
+ # the source.
167
187
  def self.newlines(source)
168
188
  Prism.parse(source).source.offsets
169
189
  end
170
-
171
- def self.parse_serialize_file(filepath)
172
- parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L"))
173
- end
174
190
  end
175
191
  end
@@ -41,23 +41,24 @@ module Prism
41
41
  # attr_reader listeners: Hash[Symbol, Array[Listener]]
42
42
  attr_reader :listeners
43
43
 
44
+ # Initialize a new dispatcher.
44
45
  def initialize
45
46
  @listeners = {}
46
47
  end
47
48
 
48
- # Register a listener for one or more events
49
+ # Register a listener for one or more events.
49
50
  #
50
51
  # def register: (Listener, *Symbol) -> void
51
52
  def register(listener, *events)
52
53
  events.each { |event| (listeners[event] ||= []) << listener }
53
54
  end
54
55
 
55
- # Walks `root` dispatching events to all registered listeners
56
+ # Walks `root` dispatching events to all registered listeners.
56
57
  #
57
58
  # def dispatch: (Node) -> void
58
59
  alias dispatch visit
59
60
 
60
- # Dispatches a single event for `node` to all registered listeners
61
+ # Dispatches a single event for `node` to all registered listeners.
61
62
  #
62
63
  # def dispatch_once: (Node) -> void
63
64
  def dispatch_once(node)
@@ -736,14 +737,6 @@ module Prism
736
737
  listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
737
738
  end
738
739
 
739
- # Dispatch enter and leave events for KeywordParameterNode nodes and continue
740
- # walking the tree.
741
- def visit_keyword_parameter_node(node)
742
- listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
743
- super
744
- listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
745
- end
746
-
747
740
  # Dispatch enter and leave events for KeywordRestParameterNode nodes and continue
748
741
  # walking the tree.
749
742
  def visit_keyword_rest_parameter_node(node)
@@ -904,6 +897,14 @@ module Prism
904
897
  listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
905
898
  end
906
899
 
900
+ # Dispatch enter and leave events for OptionalKeywordParameterNode nodes and continue
901
+ # walking the tree.
902
+ def visit_optional_keyword_parameter_node(node)
903
+ listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
904
+ super
905
+ listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
906
+ end
907
+
907
908
  # Dispatch enter and leave events for OptionalParameterNode nodes and continue
908
909
  # walking the tree.
909
910
  def visit_optional_parameter_node(node)
@@ -1008,6 +1009,14 @@ module Prism
1008
1009
  listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
1009
1010
  end
1010
1011
 
1012
+ # Dispatch enter and leave events for RequiredKeywordParameterNode nodes and continue
1013
+ # walking the tree.
1014
+ def visit_required_keyword_parameter_node(node)
1015
+ listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
1016
+ super
1017
+ listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
1018
+ end
1019
+
1011
1020
  # Dispatch enter and leave events for RequiredParameterNode nodes and continue
1012
1021
  # walking the tree.
1013
1022
  def visit_required_parameter_node(node)
@@ -1208,7 +1217,7 @@ module Prism
1208
1217
  listeners[:on_yield_node_leave]&.each { |listener| listener.on_yield_node_leave(node) }
1209
1218
  end
1210
1219
 
1211
- class DispatchOnce < Visitor
1220
+ class DispatchOnce < Visitor # :nodoc:
1212
1221
  attr_reader :listeners
1213
1222
 
1214
1223
  def initialize(listeners)
@@ -1719,12 +1728,6 @@ module Prism
1719
1728
  listeners[:on_keyword_hash_node_leave]&.each { |listener| listener.on_keyword_hash_node_leave(node) }
1720
1729
  end
1721
1730
 
1722
- # Dispatch enter and leave events for KeywordParameterNode nodes.
1723
- def visit_keyword_parameter_node(node)
1724
- listeners[:on_keyword_parameter_node_enter]&.each { |listener| listener.on_keyword_parameter_node_enter(node) }
1725
- listeners[:on_keyword_parameter_node_leave]&.each { |listener| listener.on_keyword_parameter_node_leave(node) }
1726
- end
1727
-
1728
1731
  # Dispatch enter and leave events for KeywordRestParameterNode nodes.
1729
1732
  def visit_keyword_rest_parameter_node(node)
1730
1733
  listeners[:on_keyword_rest_parameter_node_enter]&.each { |listener| listener.on_keyword_rest_parameter_node_enter(node) }
@@ -1845,6 +1848,12 @@ module Prism
1845
1848
  listeners[:on_numbered_reference_read_node_leave]&.each { |listener| listener.on_numbered_reference_read_node_leave(node) }
1846
1849
  end
1847
1850
 
1851
+ # Dispatch enter and leave events for OptionalKeywordParameterNode nodes.
1852
+ def visit_optional_keyword_parameter_node(node)
1853
+ listeners[:on_optional_keyword_parameter_node_enter]&.each { |listener| listener.on_optional_keyword_parameter_node_enter(node) }
1854
+ listeners[:on_optional_keyword_parameter_node_leave]&.each { |listener| listener.on_optional_keyword_parameter_node_leave(node) }
1855
+ end
1856
+
1848
1857
  # Dispatch enter and leave events for OptionalParameterNode nodes.
1849
1858
  def visit_optional_parameter_node(node)
1850
1859
  listeners[:on_optional_parameter_node_enter]&.each { |listener| listener.on_optional_parameter_node_enter(node) }
@@ -1923,6 +1932,12 @@ module Prism
1923
1932
  listeners[:on_regular_expression_node_leave]&.each { |listener| listener.on_regular_expression_node_leave(node) }
1924
1933
  end
1925
1934
 
1935
+ # Dispatch enter and leave events for RequiredKeywordParameterNode nodes.
1936
+ def visit_required_keyword_parameter_node(node)
1937
+ listeners[:on_required_keyword_parameter_node_enter]&.each { |listener| listener.on_required_keyword_parameter_node_enter(node) }
1938
+ listeners[:on_required_keyword_parameter_node_leave]&.each { |listener| listener.on_required_keyword_parameter_node_leave(node) }
1939
+ end
1940
+
1926
1941
  # Dispatch enter and leave events for RequiredParameterNode nodes.
1927
1942
  def visit_required_parameter_node(node)
1928
1943
  listeners[:on_required_parameter_node_enter]&.each { |listener| listener.on_required_parameter_node_enter(node) }
data/lib/prism/dsl.rb CHANGED
@@ -462,11 +462,6 @@ module Prism
462
462
  KeywordHashNode.new(elements, location)
463
463
  end
464
464
 
465
- # Create a new KeywordParameterNode node
466
- def KeywordParameterNode(name, name_loc, value, location = Location())
467
- KeywordParameterNode.new(name, name_loc, value, location)
468
- end
469
-
470
465
  # Create a new KeywordRestParameterNode node
471
466
  def KeywordRestParameterNode(name, name_loc, operator_loc, location = Location())
472
467
  KeywordRestParameterNode.new(name, name_loc, operator_loc, location)
@@ -567,6 +562,11 @@ module Prism
567
562
  NumberedReferenceReadNode.new(number, location)
568
563
  end
569
564
 
565
+ # Create a new OptionalKeywordParameterNode node
566
+ def OptionalKeywordParameterNode(name, name_loc, value, location = Location())
567
+ OptionalKeywordParameterNode.new(name, name_loc, value, location)
568
+ end
569
+
570
570
  # Create a new OptionalParameterNode node
571
571
  def OptionalParameterNode(name, name_loc, operator_loc, value, location = Location())
572
572
  OptionalParameterNode.new(name, name_loc, operator_loc, value, location)
@@ -632,6 +632,11 @@ module Prism
632
632
  RegularExpressionNode.new(opening_loc, content_loc, closing_loc, unescaped, flags, location)
633
633
  end
634
634
 
635
+ # Create a new RequiredKeywordParameterNode node
636
+ def RequiredKeywordParameterNode(name, name_loc, location = Location())
637
+ RequiredKeywordParameterNode.new(name, name_loc, location)
638
+ end
639
+
635
640
  # Create a new RequiredParameterNode node
636
641
  def RequiredParameterNode(name, location = Location())
637
642
  RequiredParameterNode.new(name, location)
data/lib/prism/ffi.rb CHANGED
@@ -9,7 +9,7 @@ require "ffi"
9
9
  module Prism
10
10
  BACKEND = :FFI
11
11
 
12
- module LibRubyParser
12
+ module LibRubyParser # :nodoc:
13
13
  extend FFI::Library
14
14
 
15
15
  # Define the library that we will be pulling functions from. Note that this
@@ -69,10 +69,10 @@ module Prism
69
69
  load_exported_functions_from(
70
70
  "prism.h",
71
71
  "pm_version",
72
- "pm_parse_serialize",
73
- "pm_parse_serialize_comments",
74
- "pm_lex_serialize",
75
- "pm_parse_lex_serialize"
72
+ "pm_serialize_parse",
73
+ "pm_serialize_parse_comments",
74
+ "pm_serialize_lex",
75
+ "pm_serialize_parse_lex"
76
76
  )
77
77
 
78
78
  load_exported_functions_from(
@@ -95,7 +95,7 @@ module Prism
95
95
 
96
96
  # This object represents a pm_buffer_t. We only use it as an opaque pointer,
97
97
  # so it doesn't need to know the fields of pm_buffer_t.
98
- class PrismBuffer
98
+ class PrismBuffer # :nodoc:
99
99
  SIZEOF = LibRubyParser.pm_buffer_sizeof
100
100
 
101
101
  attr_reader :pointer
@@ -133,7 +133,7 @@ module Prism
133
133
 
134
134
  # This object represents a pm_string_t. We only use it as an opaque pointer,
135
135
  # so it doesn't have to be an FFI::Struct.
136
- class PrismString
136
+ class PrismString # :nodoc:
137
137
  SIZEOF = LibRubyParser.pm_string_sizeof
138
138
 
139
139
  attr_reader :pointer
@@ -167,14 +167,6 @@ module Prism
167
167
  end
168
168
  end
169
169
  end
170
-
171
- def self.dump_internal(source, source_size, filepath)
172
- PrismBuffer.with do |buffer|
173
- metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
174
- pm_parse_serialize(source, source_size, buffer.pointer, metadata)
175
- buffer.read
176
- end
177
- end
178
170
  end
179
171
 
180
172
  # Mark the LibRubyParser module as private as it should only be called through
@@ -184,93 +176,153 @@ module Prism
184
176
  # The version constant is set by reading the result of calling pm_version.
185
177
  VERSION = LibRubyParser.pm_version.read_string
186
178
 
187
- # Mirror the Prism.dump API by using the serialization API.
188
- def self.dump(code, filepath = nil)
189
- LibRubyParser.dump_internal(code, code.bytesize, filepath)
190
- end
179
+ class << self
180
+ # Mirror the Prism.dump API by using the serialization API.
181
+ def dump(code, **options)
182
+ LibRubyParser::PrismBuffer.with do |buffer|
183
+ LibRubyParser.pm_serialize_parse(buffer.pointer, code, code.bytesize, dump_options(options))
184
+ buffer.read
185
+ end
186
+ end
191
187
 
192
- # Mirror the Prism.dump_file API by using the serialization API.
193
- def self.dump_file(filepath)
194
- LibRubyParser::PrismString.with(filepath) do |string|
195
- LibRubyParser.dump_internal(string.source, string.length, filepath)
188
+ # Mirror the Prism.dump_file API by using the serialization API.
189
+ def dump_file(filepath, **options)
190
+ LibRubyParser::PrismString.with(filepath) do |string|
191
+ dump(string.read, **options, filepath: filepath)
192
+ end
196
193
  end
197
- end
198
194
 
199
- # Mirror the Prism.lex API by using the serialization API.
200
- def self.lex(code, filepath = nil)
201
- LibRubyParser::PrismBuffer.with do |buffer|
202
- LibRubyParser.pm_lex_serialize(code, code.bytesize, filepath, buffer.pointer)
203
- Serialize.load_tokens(Source.new(code), buffer.read)
195
+ # Mirror the Prism.lex API by using the serialization API.
196
+ def lex(code, **options)
197
+ LibRubyParser::PrismBuffer.with do |buffer|
198
+ LibRubyParser.pm_serialize_lex(buffer.pointer, code, code.bytesize, dump_options(options))
199
+ Serialize.load_tokens(Source.new(code), buffer.read)
200
+ end
204
201
  end
205
- end
206
202
 
207
- # Mirror the Prism.lex_file API by using the serialization API.
208
- def self.lex_file(filepath)
209
- LibRubyParser::PrismString.with(filepath) do |string|
210
- lex(string.read, filepath)
203
+ # Mirror the Prism.lex_file API by using the serialization API.
204
+ def lex_file(filepath, **options)
205
+ LibRubyParser::PrismString.with(filepath) do |string|
206
+ lex(string.read, **options, filepath: filepath)
207
+ end
211
208
  end
212
- end
213
209
 
214
- # Mirror the Prism.parse API by using the serialization API.
215
- def self.parse(code, filepath = nil)
216
- Prism.load(code, dump(code, filepath))
217
- end
210
+ # Mirror the Prism.parse API by using the serialization API.
211
+ def parse(code, **options)
212
+ Prism.load(code, dump(code, **options))
213
+ end
218
214
 
219
- # Mirror the Prism.parse_file API by using the serialization API. This uses
220
- # native strings instead of Ruby strings because it allows us to use mmap when
221
- # it is available.
222
- def self.parse_file(filepath)
223
- LibRubyParser::PrismString.with(filepath) do |string|
224
- parse(string.read, filepath)
215
+ # Mirror the Prism.parse_file API by using the serialization API. This uses
216
+ # native strings instead of Ruby strings because it allows us to use mmap when
217
+ # it is available.
218
+ def parse_file(filepath, **options)
219
+ LibRubyParser::PrismString.with(filepath) do |string|
220
+ parse(string.read, **options, filepath: filepath)
221
+ end
225
222
  end
226
- end
227
223
 
228
- # Mirror the Prism.parse_comments API by using the serialization API.
229
- def self.parse_comments(code, filepath = nil)
230
- LibRubyParser::PrismBuffer.with do |buffer|
231
- metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
232
- LibRubyParser.pm_parse_serialize_comments(code, code.bytesize, buffer.pointer, metadata)
224
+ # Mirror the Prism.parse_comments API by using the serialization API.
225
+ def parse_comments(code, **options)
226
+ LibRubyParser::PrismBuffer.with do |buffer|
227
+ LibRubyParser.pm_serialize_parse_comments(buffer.pointer, code, code.bytesize, dump_options(options))
233
228
 
234
- source = Source.new(code)
235
- loader = Serialize::Loader.new(source, buffer.read)
229
+ source = Source.new(code)
230
+ loader = Serialize::Loader.new(source, buffer.read)
236
231
 
237
- loader.load_header
238
- loader.load_force_encoding
239
- loader.load_comments
232
+ loader.load_header
233
+ loader.load_force_encoding
234
+ loader.load_start_line
235
+ loader.load_comments
236
+ end
240
237
  end
241
- end
242
238
 
243
- # Mirror the Prism.parse_file_comments API by using the serialization
244
- # API. This uses native strings instead of Ruby strings because it allows us
245
- # to use mmap when it is available.
246
- def self.parse_file_comments(filepath)
247
- LibRubyParser::PrismString.with(filepath) do |string|
248
- parse_comments(string.read, filepath)
239
+ # Mirror the Prism.parse_file_comments API by using the serialization
240
+ # API. This uses native strings instead of Ruby strings because it allows us
241
+ # to use mmap when it is available.
242
+ def parse_file_comments(filepath, **options)
243
+ LibRubyParser::PrismString.with(filepath) do |string|
244
+ parse_comments(string.read, **options, filepath: filepath)
245
+ end
249
246
  end
250
- end
251
247
 
252
- # Mirror the Prism.parse_lex API by using the serialization API.
253
- def self.parse_lex(code, filepath = nil)
254
- LibRubyParser::PrismBuffer.with do |buffer|
255
- metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
256
- LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata)
248
+ # Mirror the Prism.parse_lex API by using the serialization API.
249
+ def parse_lex(code, **options)
250
+ LibRubyParser::PrismBuffer.with do |buffer|
251
+ LibRubyParser.pm_serialize_parse_lex(buffer.pointer, code, code.bytesize, dump_options(options))
257
252
 
258
- source = Source.new(code)
259
- loader = Serialize::Loader.new(source, buffer.read)
253
+ source = Source.new(code)
254
+ loader = Serialize::Loader.new(source, buffer.read)
260
255
 
261
- tokens = loader.load_tokens
262
- node, comments, magic_comments, errors, warnings = loader.load_nodes
256
+ tokens = loader.load_tokens
257
+ node, comments, magic_comments, errors, warnings = loader.load_nodes
258
+ tokens.each { |token,| token.value.force_encoding(loader.encoding) }
263
259
 
264
- tokens.each { |token,| token.value.force_encoding(loader.encoding) }
260
+ ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
261
+ end
262
+ end
265
263
 
266
- ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
264
+ # Mirror the Prism.parse_lex_file API by using the serialization API.
265
+ def parse_lex_file(filepath, **options)
266
+ LibRubyParser::PrismString.with(filepath) do |string|
267
+ parse_lex(string.read, **options, filepath: filepath)
268
+ end
267
269
  end
268
- end
269
270
 
270
- # Mirror the Prism.parse_lex_file API by using the serialization API.
271
- def self.parse_lex_file(filepath)
272
- LibRubyParser::PrismString.with(filepath) do |string|
273
- parse_lex(string.read, filepath)
271
+ private
272
+
273
+ # Convert the given options into a serialized options string.
274
+ def dump_options(options)
275
+ template = +""
276
+ values = []
277
+
278
+ template << "L"
279
+ if (filepath = options[:filepath])
280
+ values.push(filepath.bytesize, filepath.b)
281
+ template << "A*"
282
+ else
283
+ values << 0
284
+ end
285
+
286
+ template << "L"
287
+ values << options.fetch(:line, 1)
288
+
289
+ template << "L"
290
+ if (encoding = options[:encoding])
291
+ name = encoding.name
292
+ values.push(name.bytesize, name.b)
293
+ template << "A*"
294
+ else
295
+ values << 0
296
+ end
297
+
298
+ template << "C"
299
+ values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
300
+
301
+ template << "C"
302
+ values << (options[:verbose] ? 0 : 1)
303
+
304
+ template << "L"
305
+ if (scopes = options[:scopes])
306
+ values << scopes.length
307
+
308
+ scopes.each do |scope|
309
+ template << "L"
310
+ values << scope.length
311
+
312
+ scope.each do |local|
313
+ name = local.name
314
+ template << "L"
315
+ values << name.bytesize
316
+
317
+ template << "A*"
318
+ values << name.b
319
+ end
320
+ end
321
+ else
322
+ values << 0
323
+ end
324
+
325
+ values.pack(template)
274
326
  end
275
327
  end
276
328
  end
@@ -8,7 +8,7 @@ module Prism
8
8
  # of cases, this is a one-to-one mapping of the token type. Everything else
9
9
  # generally lines up. However, there are a few cases that require special
10
10
  # handling.
11
- class LexCompat
11
+ class LexCompat # :nodoc:
12
12
  # This is a mapping of prism token types to Ripper token types. This is a
13
13
  # many-to-one mapping because we split up our token types, whereas Ripper
14
14
  # tends to group them.
@@ -184,18 +184,22 @@ module Prism
184
184
  # However, we add a couple of convenience methods onto them to make them a
185
185
  # little easier to work with. We delegate all other methods to the array.
186
186
  class Token < SimpleDelegator
187
+ # The location of the token in the source.
187
188
  def location
188
189
  self[0]
189
190
  end
190
191
 
192
+ # The type of the token.
191
193
  def event
192
194
  self[1]
193
195
  end
194
196
 
197
+ # The slice of the source that this token represents.
195
198
  def value
196
199
  self[2]
197
200
  end
198
201
 
202
+ # The state of the lexer when this token was produced.
199
203
  def state
200
204
  self[3]
201
205
  end
@@ -204,7 +208,7 @@ module Prism
204
208
  # Ripper doesn't include the rest of the token in the event, so we need to
205
209
  # trim it down to just the content on the first line when comparing.
206
210
  class EndContentToken < Token
207
- def ==(other)
211
+ def ==(other) # :nodoc:
208
212
  [self[0], self[1], self[2][0..self[2].index("\n")], self[3]] == other
209
213
  end
210
214
  end
@@ -212,7 +216,7 @@ module Prism
212
216
  # Tokens where state should be ignored
213
217
  # used for :on_comment, :on_heredoc_end, :on_embexpr_end
214
218
  class IgnoreStateToken < Token
215
- def ==(other)
219
+ def ==(other) # :nodoc:
216
220
  self[0...-1] == other[0...-1]
217
221
  end
218
222
  end
@@ -222,7 +226,7 @@ module Prism
222
226
  # through named captures in regular expressions). In that case we don't
223
227
  # compare the state.
224
228
  class IdentToken < Token
225
- def ==(other)
229
+ def ==(other) # :nodoc:
226
230
  (self[0...-1] == other[0...-1]) && (
227
231
  (other[3] == Ripper::EXPR_LABEL | Ripper::EXPR_END) ||
228
232
  (other[3] & Ripper::EXPR_ARG_ANY != 0)
@@ -233,7 +237,7 @@ module Prism
233
237
  # Ignored newlines can occasionally have a LABEL state attached to them, so
234
238
  # we compare the state differently here.
235
239
  class IgnoredNewlineToken < Token
236
- def ==(other)
240
+ def ==(other) # :nodoc:
237
241
  return false unless self[0...-1] == other[0...-1]
238
242
 
239
243
  if self[4] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED
@@ -253,7 +257,7 @@ module Prism
253
257
  # more accurately, so we need to allow comparing against both END and
254
258
  # END|LABEL.
255
259
  class ParamToken < Token
256
- def ==(other)
260
+ def ==(other) # :nodoc:
257
261
  (self[0...-1] == other[0...-1]) && (
258
262
  (other[3] == Ripper::EXPR_END) ||
259
263
  (other[3] == Ripper::EXPR_END | Ripper::EXPR_LABEL)
@@ -264,12 +268,12 @@ module Prism
264
268
  # A heredoc in this case is a list of tokens that belong to the body of the
265
269
  # heredoc that should be appended onto the list of tokens when the heredoc
266
270
  # closes.
267
- module Heredoc
271
+ module Heredoc # :nodoc:
268
272
  # Heredocs that are no dash or tilde heredocs are just a list of tokens.
269
273
  # We need to keep them around so that we can insert them in the correct
270
274
  # order back into the token stream and set the state of the last token to
271
275
  # the state that the heredoc was opened in.
272
- class PlainHeredoc
276
+ class PlainHeredoc # :nodoc:
273
277
  attr_reader :tokens
274
278
 
275
279
  def initialize
@@ -288,7 +292,7 @@ module Prism
288
292
  # Dash heredocs are a little more complicated. They are a list of tokens
289
293
  # that need to be split on "\\\n" to mimic Ripper's behavior. We also need
290
294
  # to keep track of the state that the heredoc was opened in.
291
- class DashHeredoc
295
+ class DashHeredoc # :nodoc:
292
296
  attr_reader :split, :tokens
293
297
 
294
298
  def initialize(split)
@@ -347,7 +351,7 @@ module Prism
347
351
  # insert them into the stream in the correct order. As such, we can do
348
352
  # some extra manipulation on the tokens to make them match Ripper's
349
353
  # output by mirroring the dedent logic that Ripper uses.
350
- class DedentingHeredoc
354
+ class DedentingHeredoc # :nodoc:
351
355
  TAB_WIDTH = 8
352
356
 
353
357
  attr_reader :tokens, :dedent_next, :dedent, :embexpr_balance
@@ -588,11 +592,13 @@ module Prism
588
592
  end
589
593
  end
590
594
 
591
- attr_reader :source, :filepath
595
+ private_constant :Heredoc
592
596
 
593
- def initialize(source, filepath = "")
597
+ attr_reader :source, :options
598
+
599
+ def initialize(source, **options)
594
600
  @source = source
595
- @filepath = filepath || ""
601
+ @options = options
596
602
  end
597
603
 
598
604
  def result
@@ -601,7 +607,7 @@ module Prism
601
607
  state = :default
602
608
  heredoc_stack = [[]]
603
609
 
604
- result = Prism.lex(source, @filepath)
610
+ result = Prism.lex(source, **options)
605
611
  result_value = result.value
606
612
  previous_state = nil
607
613
 
@@ -829,9 +835,11 @@ module Prism
829
835
  end
830
836
  end
831
837
 
838
+ private_constant :LexCompat
839
+
832
840
  # This is a class that wraps the Ripper lexer to produce almost exactly the
833
841
  # same tokens.
834
- class LexRipper
842
+ class LexRipper # :nodoc:
835
843
  attr_reader :source
836
844
 
837
845
  def initialize(source)
@@ -869,4 +877,6 @@ module Prism
869
877
  results
870
878
  end
871
879
  end
880
+
881
+ private_constant :LexRipper
872
882
  end
@@ -430,11 +430,6 @@ module Prism
430
430
  node.copy(elements: visit_all(node.elements))
431
431
  end
432
432
 
433
- # Copy a KeywordParameterNode node
434
- def visit_keyword_parameter_node(node)
435
- node.copy(value: visit(node.value))
436
- end
437
-
438
433
  # Copy a KeywordRestParameterNode node
439
434
  def visit_keyword_rest_parameter_node(node)
440
435
  node.copy
@@ -535,6 +530,11 @@ module Prism
535
530
  node.copy
536
531
  end
537
532
 
533
+ # Copy a OptionalKeywordParameterNode node
534
+ def visit_optional_keyword_parameter_node(node)
535
+ node.copy(value: visit(node.value))
536
+ end
537
+
538
538
  # Copy a OptionalParameterNode node
539
539
  def visit_optional_parameter_node(node)
540
540
  node.copy(value: visit(node.value))
@@ -600,6 +600,11 @@ module Prism
600
600
  node.copy
601
601
  end
602
602
 
603
+ # Copy a RequiredKeywordParameterNode node
604
+ def visit_required_keyword_parameter_node(node)
605
+ node.copy
606
+ end
607
+
603
608
  # Copy a RequiredParameterNode node
604
609
  def visit_required_parameter_node(node)
605
610
  node.copy