yarp 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +7 -0
  2. data/CODE_OF_CONDUCT.md +76 -0
  3. data/CONTRIBUTING.md +51 -0
  4. data/LICENSE.md +7 -0
  5. data/Makefile.in +79 -0
  6. data/README.md +86 -0
  7. data/config.h.in +25 -0
  8. data/config.yml +2147 -0
  9. data/configure +4487 -0
  10. data/docs/build_system.md +85 -0
  11. data/docs/building.md +26 -0
  12. data/docs/configuration.md +56 -0
  13. data/docs/design.md +53 -0
  14. data/docs/encoding.md +116 -0
  15. data/docs/extension.md +20 -0
  16. data/docs/fuzzing.md +93 -0
  17. data/docs/heredocs.md +36 -0
  18. data/docs/mapping.md +117 -0
  19. data/docs/ripper.md +36 -0
  20. data/docs/serialization.md +130 -0
  21. data/docs/testing.md +55 -0
  22. data/ext/yarp/api_node.c +3680 -0
  23. data/ext/yarp/api_pack.c +256 -0
  24. data/ext/yarp/extconf.rb +131 -0
  25. data/ext/yarp/extension.c +547 -0
  26. data/ext/yarp/extension.h +18 -0
  27. data/include/yarp/ast.h +1412 -0
  28. data/include/yarp/defines.h +54 -0
  29. data/include/yarp/diagnostic.h +24 -0
  30. data/include/yarp/enc/yp_encoding.h +94 -0
  31. data/include/yarp/node.h +36 -0
  32. data/include/yarp/pack.h +141 -0
  33. data/include/yarp/parser.h +389 -0
  34. data/include/yarp/regexp.h +19 -0
  35. data/include/yarp/unescape.h +42 -0
  36. data/include/yarp/util/yp_buffer.h +39 -0
  37. data/include/yarp/util/yp_char.h +75 -0
  38. data/include/yarp/util/yp_constant_pool.h +64 -0
  39. data/include/yarp/util/yp_list.h +67 -0
  40. data/include/yarp/util/yp_memchr.h +14 -0
  41. data/include/yarp/util/yp_newline_list.h +54 -0
  42. data/include/yarp/util/yp_state_stack.h +24 -0
  43. data/include/yarp/util/yp_string.h +57 -0
  44. data/include/yarp/util/yp_string_list.h +28 -0
  45. data/include/yarp/util/yp_strpbrk.h +29 -0
  46. data/include/yarp/version.h +5 -0
  47. data/include/yarp.h +69 -0
  48. data/lib/yarp/lex_compat.rb +759 -0
  49. data/lib/yarp/node.rb +7428 -0
  50. data/lib/yarp/pack.rb +185 -0
  51. data/lib/yarp/ripper_compat.rb +174 -0
  52. data/lib/yarp/serialize.rb +389 -0
  53. data/lib/yarp.rb +330 -0
  54. data/src/diagnostic.c +25 -0
  55. data/src/enc/yp_big5.c +79 -0
  56. data/src/enc/yp_euc_jp.c +85 -0
  57. data/src/enc/yp_gbk.c +88 -0
  58. data/src/enc/yp_shift_jis.c +83 -0
  59. data/src/enc/yp_tables.c +509 -0
  60. data/src/enc/yp_unicode.c +2320 -0
  61. data/src/enc/yp_windows_31j.c +83 -0
  62. data/src/node.c +2011 -0
  63. data/src/pack.c +493 -0
  64. data/src/prettyprint.c +1782 -0
  65. data/src/regexp.c +580 -0
  66. data/src/serialize.c +1576 -0
  67. data/src/token_type.c +347 -0
  68. data/src/unescape.c +576 -0
  69. data/src/util/yp_buffer.c +78 -0
  70. data/src/util/yp_char.c +229 -0
  71. data/src/util/yp_constant_pool.c +147 -0
  72. data/src/util/yp_list.c +50 -0
  73. data/src/util/yp_memchr.c +31 -0
  74. data/src/util/yp_newline_list.c +119 -0
  75. data/src/util/yp_state_stack.c +25 -0
  76. data/src/util/yp_string.c +207 -0
  77. data/src/util/yp_string_list.c +32 -0
  78. data/src/util/yp_strncasecmp.c +20 -0
  79. data/src/util/yp_strpbrk.c +66 -0
  80. data/src/yarp.c +13211 -0
  81. data/yarp.gemspec +100 -0
  82. metadata +125 -0
@@ -0,0 +1,389 @@
1
+ # frozen_string_literal: true
2
+ =begin
3
+ This file is generated by the bin/template script and should not be
4
+ modified manually. See templates/lib/yarp/serialize.rb.erb
5
+ if you are looking to modify the template
6
+ =end
7
+
8
+ require "stringio"
9
+
10
+ # Polyfill for String#unpack1 with the offset parameter.
11
+ if String.instance_method(:unpack1).parameters.none? { |_, name| name == :offset }
12
+ String.prepend(
13
+ Module.new {
14
+ def unpack1(format, offset: 0)
15
+ offset == 0 ? super(format) : self[offset..].unpack1(format)
16
+ end
17
+ }
18
+ )
19
+ end
20
+
21
+ module YARP
22
+ module Serialize
23
+ def self.load(input, serialized)
24
+ io = StringIO.new(serialized)
25
+ io.set_encoding(Encoding::BINARY)
26
+
27
+ Loader.new(input, serialized, io).load
28
+ end
29
+
30
+ class Loader
31
+ attr_reader :encoding, :input, :serialized, :io
32
+ attr_reader :constant_pool_offset, :constant_pool, :source
33
+
34
+ def initialize(input, serialized, io)
35
+ @encoding = Encoding::UTF_8
36
+
37
+ @input = input.dup
38
+ @serialized = serialized
39
+ @io = io
40
+
41
+ @constant_pool_offset = nil
42
+ @constant_pool = nil
43
+
44
+ offsets = [0]
45
+ input.b.scan("\n") { offsets << $~.end(0) }
46
+ @source = Source.new(input, offsets)
47
+ end
48
+
49
+ def load
50
+ raise "Invalid serialization" if io.read(4) != "YARP"
51
+ raise "Invalid serialization" if io.read(3).unpack("C3") != [0, 6, 0]
52
+
53
+ @encoding = Encoding.find(io.read(load_varint))
54
+ @input = input.force_encoding(@encoding).freeze
55
+
56
+ errors = load_varint.times.map { ParseError.new(load_string, load_location) }
57
+ warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) }
58
+
59
+ @constant_pool_offset = io.read(4).unpack1("L")
60
+ @constant_pool = Array.new(load_varint, nil)
61
+
62
+ ast = load_node
63
+
64
+ YARP::ParseResult.new(ast, [], errors, warnings, @source)
65
+ end
66
+
67
+ private
68
+
69
+ # variable-length integer using https://en.wikipedia.org/wiki/LEB128
70
+ # This is also what protobuf uses: https://protobuf.dev/programming-guides/encoding/#varints
71
+ def load_varint
72
+ n = io.getbyte
73
+ if n < 128
74
+ n
75
+ else
76
+ n -= 128
77
+ shift = 0
78
+ while (b = io.getbyte) >= 128
79
+ n += (b - 128) << (shift += 7)
80
+ end
81
+ n + (b << (shift + 7))
82
+ end
83
+ end
84
+
85
+ def load_serialized_length
86
+ io.read(4).unpack1("L")
87
+ end
88
+
89
+ def load_optional_node
90
+ if io.getbyte != 0
91
+ io.pos -= 1
92
+ load_node
93
+ end
94
+ end
95
+
96
+ def load_string
97
+ io.read(load_varint).force_encoding(encoding)
98
+ end
99
+
100
+ def load_location
101
+ Location.new(source, load_varint, load_varint)
102
+ end
103
+
104
+ def load_optional_location
105
+ load_location if io.getbyte != 0
106
+ end
107
+
108
+ def load_constant
109
+ index = load_varint - 1
110
+ constant = constant_pool[index]
111
+
112
+ unless constant
113
+ offset = constant_pool_offset + index * 8
114
+
115
+ start = serialized.unpack1("L", offset: offset)
116
+ length = serialized.unpack1("L", offset: offset + 4)
117
+
118
+ constant = input.byteslice(start, length).to_sym
119
+ constant_pool[index] = constant
120
+ end
121
+
122
+ constant
123
+ end
124
+
125
+ def load_node
126
+ type = io.getbyte
127
+ location = load_location
128
+
129
+ case type
130
+ when 1 then
131
+ AliasNode.new(load_node, load_node, load_location, location)
132
+ when 2 then
133
+ AlternationPatternNode.new(load_node, load_node, load_location, location)
134
+ when 3 then
135
+ AndNode.new(load_node, load_node, load_location, location)
136
+ when 4 then
137
+ ArgumentsNode.new(Array.new(load_varint) { load_node }, location)
138
+ when 5 then
139
+ ArrayNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
140
+ when 6 then
141
+ ArrayPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
142
+ when 7 then
143
+ AssocNode.new(load_node, load_optional_node, load_optional_location, location)
144
+ when 8 then
145
+ AssocSplatNode.new(load_optional_node, load_location, location)
146
+ when 9 then
147
+ BackReferenceReadNode.new(location)
148
+ when 10 then
149
+ BeginNode.new(load_optional_location, load_optional_node, load_optional_node, load_optional_node, load_optional_node, load_optional_location, location)
150
+ when 11 then
151
+ BlockArgumentNode.new(load_optional_node, load_location, location)
152
+ when 12 then
153
+ BlockNode.new(Array.new(load_varint) { load_constant }, load_optional_node, load_optional_node, load_location, load_location, location)
154
+ when 13 then
155
+ BlockParameterNode.new(load_optional_location, load_location, location)
156
+ when 14 then
157
+ BlockParametersNode.new(load_optional_node, Array.new(load_varint) { load_location }, load_optional_location, load_optional_location, location)
158
+ when 15 then
159
+ BreakNode.new(load_optional_node, load_location, location)
160
+ when 16 then
161
+ CallNode.new(load_optional_node, load_optional_location, load_optional_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, load_varint, load_string, location)
162
+ when 17 then
163
+ CallOperatorAndWriteNode.new(load_node, load_location, load_node, location)
164
+ when 18 then
165
+ CallOperatorOrWriteNode.new(load_node, load_node, load_location, location)
166
+ when 19 then
167
+ CallOperatorWriteNode.new(load_node, load_location, load_node, load_constant, location)
168
+ when 20 then
169
+ CapturePatternNode.new(load_node, load_node, load_location, location)
170
+ when 21 then
171
+ CaseNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_location, load_location, location)
172
+ when 22 then
173
+ ClassNode.new(Array.new(load_varint) { load_constant }, load_location, load_node, load_optional_location, load_optional_node, load_optional_node, load_location, location)
174
+ when 23 then
175
+ ClassVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
176
+ when 24 then
177
+ ClassVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
178
+ when 25 then
179
+ ClassVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
180
+ when 26 then
181
+ ClassVariableReadNode.new(location)
182
+ when 27 then
183
+ ClassVariableWriteNode.new(load_location, load_optional_node, load_optional_location, location)
184
+ when 28 then
185
+ ConstantOperatorAndWriteNode.new(load_location, load_location, load_node, location)
186
+ when 29 then
187
+ ConstantOperatorOrWriteNode.new(load_location, load_location, load_node, location)
188
+ when 30 then
189
+ ConstantOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
190
+ when 31 then
191
+ ConstantPathNode.new(load_optional_node, load_node, load_location, location)
192
+ when 32 then
193
+ ConstantPathOperatorAndWriteNode.new(load_node, load_location, load_node, location)
194
+ when 33 then
195
+ ConstantPathOperatorOrWriteNode.new(load_node, load_location, load_node, location)
196
+ when 34 then
197
+ ConstantPathOperatorWriteNode.new(load_node, load_location, load_node, load_constant, location)
198
+ when 35 then
199
+ ConstantPathWriteNode.new(load_node, load_optional_location, load_optional_node, location)
200
+ when 36 then
201
+ ConstantReadNode.new(location)
202
+ when 37 then
203
+ ConstantWriteNode.new(load_location, load_optional_node, load_optional_location, location)
204
+ when 38 then
205
+ load_serialized_length
206
+ DefNode.new(load_location, load_optional_node, load_optional_node, load_optional_node, Array.new(load_varint) { load_constant }, load_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, location)
207
+ when 39 then
208
+ DefinedNode.new(load_optional_location, load_node, load_optional_location, load_location, location)
209
+ when 40 then
210
+ ElseNode.new(load_location, load_optional_node, load_optional_location, location)
211
+ when 41 then
212
+ EmbeddedStatementsNode.new(load_location, load_optional_node, load_location, location)
213
+ when 42 then
214
+ EmbeddedVariableNode.new(load_location, load_node, location)
215
+ when 43 then
216
+ EnsureNode.new(load_location, load_optional_node, load_location, location)
217
+ when 44 then
218
+ FalseNode.new(location)
219
+ when 45 then
220
+ FindPatternNode.new(load_optional_node, load_node, Array.new(load_varint) { load_node }, load_node, load_optional_location, load_optional_location, location)
221
+ when 46 then
222
+ FloatNode.new(location)
223
+ when 47 then
224
+ ForNode.new(load_node, load_node, load_optional_node, load_location, load_location, load_optional_location, load_location, location)
225
+ when 48 then
226
+ ForwardingArgumentsNode.new(location)
227
+ when 49 then
228
+ ForwardingParameterNode.new(location)
229
+ when 50 then
230
+ ForwardingSuperNode.new(load_optional_node, location)
231
+ when 51 then
232
+ GlobalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
233
+ when 52 then
234
+ GlobalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
235
+ when 53 then
236
+ GlobalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
237
+ when 54 then
238
+ GlobalVariableReadNode.new(location)
239
+ when 55 then
240
+ GlobalVariableWriteNode.new(load_location, load_optional_location, load_optional_node, location)
241
+ when 56 then
242
+ HashNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
243
+ when 57 then
244
+ HashPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_location, load_optional_location, location)
245
+ when 58 then
246
+ IfNode.new(load_optional_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
247
+ when 59 then
248
+ ImaginaryNode.new(load_node, location)
249
+ when 60 then
250
+ InNode.new(load_node, load_optional_node, load_location, load_optional_location, location)
251
+ when 61 then
252
+ InstanceVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
253
+ when 62 then
254
+ InstanceVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
255
+ when 63 then
256
+ InstanceVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
257
+ when 64 then
258
+ InstanceVariableReadNode.new(location)
259
+ when 65 then
260
+ InstanceVariableWriteNode.new(load_location, load_optional_node, load_optional_location, location)
261
+ when 66 then
262
+ IntegerNode.new(location)
263
+ when 67 then
264
+ InterpolatedRegularExpressionNode.new(load_location, Array.new(load_varint) { load_node }, load_location, load_varint, location)
265
+ when 68 then
266
+ InterpolatedStringNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
267
+ when 69 then
268
+ InterpolatedSymbolNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
269
+ when 70 then
270
+ InterpolatedXStringNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
271
+ when 71 then
272
+ KeywordHashNode.new(Array.new(load_varint) { load_node }, location)
273
+ when 72 then
274
+ KeywordParameterNode.new(load_location, load_optional_node, location)
275
+ when 73 then
276
+ KeywordRestParameterNode.new(load_location, load_optional_location, location)
277
+ when 74 then
278
+ LambdaNode.new(Array.new(load_varint) { load_constant }, load_location, load_optional_node, load_optional_node, location)
279
+ when 75 then
280
+ LocalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, load_constant, location)
281
+ when 76 then
282
+ LocalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, load_constant, location)
283
+ when 77 then
284
+ LocalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, load_constant, location)
285
+ when 78 then
286
+ LocalVariableReadNode.new(load_constant, load_varint, location)
287
+ when 79 then
288
+ LocalVariableWriteNode.new(load_constant, load_varint, load_optional_node, load_location, load_optional_location, location)
289
+ when 80 then
290
+ MatchPredicateNode.new(load_node, load_node, load_location, location)
291
+ when 81 then
292
+ MatchRequiredNode.new(load_node, load_node, load_location, location)
293
+ when 82 then
294
+ MissingNode.new(location)
295
+ when 83 then
296
+ ModuleNode.new(Array.new(load_varint) { load_constant }, load_location, load_node, load_optional_node, load_location, location)
297
+ when 84 then
298
+ MultiWriteNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_location, load_optional_location, location)
299
+ when 85 then
300
+ NextNode.new(load_optional_node, load_location, location)
301
+ when 86 then
302
+ NilNode.new(location)
303
+ when 87 then
304
+ NoKeywordsParameterNode.new(load_location, load_location, location)
305
+ when 88 then
306
+ NumberedReferenceReadNode.new(location)
307
+ when 89 then
308
+ OptionalParameterNode.new(load_constant, load_location, load_location, load_node, location)
309
+ when 90 then
310
+ OrNode.new(load_node, load_node, load_location, location)
311
+ when 91 then
312
+ ParametersNode.new(Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_node, location)
313
+ when 92 then
314
+ ParenthesesNode.new(load_optional_node, load_location, load_location, location)
315
+ when 93 then
316
+ PinnedExpressionNode.new(load_node, load_location, load_location, load_location, location)
317
+ when 94 then
318
+ PinnedVariableNode.new(load_node, load_location, location)
319
+ when 95 then
320
+ PostExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
321
+ when 96 then
322
+ PreExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
323
+ when 97 then
324
+ ProgramNode.new(Array.new(load_varint) { load_constant }, load_node, location)
325
+ when 98 then
326
+ RangeNode.new(load_optional_node, load_optional_node, load_location, load_varint, location)
327
+ when 99 then
328
+ RationalNode.new(load_node, location)
329
+ when 100 then
330
+ RedoNode.new(location)
331
+ when 101 then
332
+ RegularExpressionNode.new(load_location, load_location, load_location, load_string, load_varint, location)
333
+ when 102 then
334
+ RequiredDestructuredParameterNode.new(Array.new(load_varint) { load_node }, load_location, load_location, location)
335
+ when 103 then
336
+ RequiredParameterNode.new(load_constant, location)
337
+ when 104 then
338
+ RescueModifierNode.new(load_node, load_location, load_node, location)
339
+ when 105 then
340
+ RescueNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_node, load_optional_node, location)
341
+ when 106 then
342
+ RestParameterNode.new(load_location, load_optional_location, location)
343
+ when 107 then
344
+ RetryNode.new(location)
345
+ when 108 then
346
+ ReturnNode.new(load_location, load_optional_node, location)
347
+ when 109 then
348
+ SelfNode.new(location)
349
+ when 110 then
350
+ SingletonClassNode.new(Array.new(load_varint) { load_constant }, load_location, load_location, load_node, load_optional_node, load_location, location)
351
+ when 111 then
352
+ SourceEncodingNode.new(location)
353
+ when 112 then
354
+ SourceFileNode.new(load_string, location)
355
+ when 113 then
356
+ SourceLineNode.new(location)
357
+ when 114 then
358
+ SplatNode.new(load_location, load_optional_node, location)
359
+ when 115 then
360
+ StatementsNode.new(Array.new(load_varint) { load_node }, location)
361
+ when 116 then
362
+ StringConcatNode.new(load_node, load_node, location)
363
+ when 117 then
364
+ StringNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
365
+ when 118 then
366
+ SuperNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, location)
367
+ when 119 then
368
+ SymbolNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
369
+ when 120 then
370
+ TrueNode.new(location)
371
+ when 121 then
372
+ UndefNode.new(Array.new(load_varint) { load_node }, load_location, location)
373
+ when 122 then
374
+ UnlessNode.new(load_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
375
+ when 123 then
376
+ UntilNode.new(load_location, load_node, load_optional_node, load_varint, location)
377
+ when 124 then
378
+ WhenNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_node, location)
379
+ when 125 then
380
+ WhileNode.new(load_location, load_node, load_optional_node, load_varint, location)
381
+ when 126 then
382
+ XStringNode.new(load_location, load_location, load_location, load_string, location)
383
+ when 127 then
384
+ YieldNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, location)
385
+ end
386
+ end
387
+ end
388
+ end
389
+ end