yarp 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (82) hide show
  1. checksums.yaml +7 -0
  2. data/CODE_OF_CONDUCT.md +76 -0
  3. data/CONTRIBUTING.md +51 -0
  4. data/LICENSE.md +7 -0
  5. data/Makefile.in +79 -0
  6. data/README.md +86 -0
  7. data/config.h.in +25 -0
  8. data/config.yml +2147 -0
  9. data/configure +4487 -0
  10. data/docs/build_system.md +85 -0
  11. data/docs/building.md +26 -0
  12. data/docs/configuration.md +56 -0
  13. data/docs/design.md +53 -0
  14. data/docs/encoding.md +116 -0
  15. data/docs/extension.md +20 -0
  16. data/docs/fuzzing.md +93 -0
  17. data/docs/heredocs.md +36 -0
  18. data/docs/mapping.md +117 -0
  19. data/docs/ripper.md +36 -0
  20. data/docs/serialization.md +130 -0
  21. data/docs/testing.md +55 -0
  22. data/ext/yarp/api_node.c +3680 -0
  23. data/ext/yarp/api_pack.c +256 -0
  24. data/ext/yarp/extconf.rb +131 -0
  25. data/ext/yarp/extension.c +547 -0
  26. data/ext/yarp/extension.h +18 -0
  27. data/include/yarp/ast.h +1412 -0
  28. data/include/yarp/defines.h +54 -0
  29. data/include/yarp/diagnostic.h +24 -0
  30. data/include/yarp/enc/yp_encoding.h +94 -0
  31. data/include/yarp/node.h +36 -0
  32. data/include/yarp/pack.h +141 -0
  33. data/include/yarp/parser.h +389 -0
  34. data/include/yarp/regexp.h +19 -0
  35. data/include/yarp/unescape.h +42 -0
  36. data/include/yarp/util/yp_buffer.h +39 -0
  37. data/include/yarp/util/yp_char.h +75 -0
  38. data/include/yarp/util/yp_constant_pool.h +64 -0
  39. data/include/yarp/util/yp_list.h +67 -0
  40. data/include/yarp/util/yp_memchr.h +14 -0
  41. data/include/yarp/util/yp_newline_list.h +54 -0
  42. data/include/yarp/util/yp_state_stack.h +24 -0
  43. data/include/yarp/util/yp_string.h +57 -0
  44. data/include/yarp/util/yp_string_list.h +28 -0
  45. data/include/yarp/util/yp_strpbrk.h +29 -0
  46. data/include/yarp/version.h +5 -0
  47. data/include/yarp.h +69 -0
  48. data/lib/yarp/lex_compat.rb +759 -0
  49. data/lib/yarp/node.rb +7428 -0
  50. data/lib/yarp/pack.rb +185 -0
  51. data/lib/yarp/ripper_compat.rb +174 -0
  52. data/lib/yarp/serialize.rb +389 -0
  53. data/lib/yarp.rb +330 -0
  54. data/src/diagnostic.c +25 -0
  55. data/src/enc/yp_big5.c +79 -0
  56. data/src/enc/yp_euc_jp.c +85 -0
  57. data/src/enc/yp_gbk.c +88 -0
  58. data/src/enc/yp_shift_jis.c +83 -0
  59. data/src/enc/yp_tables.c +509 -0
  60. data/src/enc/yp_unicode.c +2320 -0
  61. data/src/enc/yp_windows_31j.c +83 -0
  62. data/src/node.c +2011 -0
  63. data/src/pack.c +493 -0
  64. data/src/prettyprint.c +1782 -0
  65. data/src/regexp.c +580 -0
  66. data/src/serialize.c +1576 -0
  67. data/src/token_type.c +347 -0
  68. data/src/unescape.c +576 -0
  69. data/src/util/yp_buffer.c +78 -0
  70. data/src/util/yp_char.c +229 -0
  71. data/src/util/yp_constant_pool.c +147 -0
  72. data/src/util/yp_list.c +50 -0
  73. data/src/util/yp_memchr.c +31 -0
  74. data/src/util/yp_newline_list.c +119 -0
  75. data/src/util/yp_state_stack.c +25 -0
  76. data/src/util/yp_string.c +207 -0
  77. data/src/util/yp_string_list.c +32 -0
  78. data/src/util/yp_strncasecmp.c +20 -0
  79. data/src/util/yp_strpbrk.c +66 -0
  80. data/src/yarp.c +13211 -0
  81. data/yarp.gemspec +100 -0
  82. metadata +125 -0
@@ -0,0 +1,389 @@
1
+ # frozen_string_literal: true
2
+ =begin
3
+ This file is generated by the bin/template script and should not be
4
+ modified manually. See templates/lib/yarp/serialize.rb.erb
5
+ if you are looking to modify the template
6
+ =end
7
+
8
+ require "stringio"
9
+
10
+ # Polyfill for String#unpack1 with the offset parameter.
11
+ if String.instance_method(:unpack1).parameters.none? { |_, name| name == :offset }
12
+ String.prepend(
13
+ Module.new {
14
+ def unpack1(format, offset: 0)
15
+ offset == 0 ? super(format) : self[offset..].unpack1(format)
16
+ end
17
+ }
18
+ )
19
+ end
20
+
21
+ module YARP
22
+ module Serialize
23
+ def self.load(input, serialized)
24
+ io = StringIO.new(serialized)
25
+ io.set_encoding(Encoding::BINARY)
26
+
27
+ Loader.new(input, serialized, io).load
28
+ end
29
+
30
+ class Loader
31
+ attr_reader :encoding, :input, :serialized, :io
32
+ attr_reader :constant_pool_offset, :constant_pool, :source
33
+
34
+ def initialize(input, serialized, io)
35
+ @encoding = Encoding::UTF_8
36
+
37
+ @input = input.dup
38
+ @serialized = serialized
39
+ @io = io
40
+
41
+ @constant_pool_offset = nil
42
+ @constant_pool = nil
43
+
44
+ offsets = [0]
45
+ input.b.scan("\n") { offsets << $~.end(0) }
46
+ @source = Source.new(input, offsets)
47
+ end
48
+
49
+ def load
50
+ raise "Invalid serialization" if io.read(4) != "YARP"
51
+ raise "Invalid serialization" if io.read(3).unpack("C3") != [0, 6, 0]
52
+
53
+ @encoding = Encoding.find(io.read(load_varint))
54
+ @input = input.force_encoding(@encoding).freeze
55
+
56
+ errors = load_varint.times.map { ParseError.new(load_string, load_location) }
57
+ warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) }
58
+
59
+ @constant_pool_offset = io.read(4).unpack1("L")
60
+ @constant_pool = Array.new(load_varint, nil)
61
+
62
+ ast = load_node
63
+
64
+ YARP::ParseResult.new(ast, [], errors, warnings, @source)
65
+ end
66
+
67
+ private
68
+
69
+ # variable-length integer using https://en.wikipedia.org/wiki/LEB128
70
+ # This is also what protobuf uses: https://protobuf.dev/programming-guides/encoding/#varints
71
+ def load_varint
72
+ n = io.getbyte
73
+ if n < 128
74
+ n
75
+ else
76
+ n -= 128
77
+ shift = 0
78
+ while (b = io.getbyte) >= 128
79
+ n += (b - 128) << (shift += 7)
80
+ end
81
+ n + (b << (shift + 7))
82
+ end
83
+ end
84
+
85
+ def load_serialized_length
86
+ io.read(4).unpack1("L")
87
+ end
88
+
89
+ def load_optional_node
90
+ if io.getbyte != 0
91
+ io.pos -= 1
92
+ load_node
93
+ end
94
+ end
95
+
96
+ def load_string
97
+ io.read(load_varint).force_encoding(encoding)
98
+ end
99
+
100
+ def load_location
101
+ Location.new(source, load_varint, load_varint)
102
+ end
103
+
104
+ def load_optional_location
105
+ load_location if io.getbyte != 0
106
+ end
107
+
108
+ def load_constant
109
+ index = load_varint - 1
110
+ constant = constant_pool[index]
111
+
112
+ unless constant
113
+ offset = constant_pool_offset + index * 8
114
+
115
+ start = serialized.unpack1("L", offset: offset)
116
+ length = serialized.unpack1("L", offset: offset + 4)
117
+
118
+ constant = input.byteslice(start, length).to_sym
119
+ constant_pool[index] = constant
120
+ end
121
+
122
+ constant
123
+ end
124
+
125
+ def load_node
126
+ type = io.getbyte
127
+ location = load_location
128
+
129
+ case type
130
+ when 1 then
131
+ AliasNode.new(load_node, load_node, load_location, location)
132
+ when 2 then
133
+ AlternationPatternNode.new(load_node, load_node, load_location, location)
134
+ when 3 then
135
+ AndNode.new(load_node, load_node, load_location, location)
136
+ when 4 then
137
+ ArgumentsNode.new(Array.new(load_varint) { load_node }, location)
138
+ when 5 then
139
+ ArrayNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
140
+ when 6 then
141
+ ArrayPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
142
+ when 7 then
143
+ AssocNode.new(load_node, load_optional_node, load_optional_location, location)
144
+ when 8 then
145
+ AssocSplatNode.new(load_optional_node, load_location, location)
146
+ when 9 then
147
+ BackReferenceReadNode.new(location)
148
+ when 10 then
149
+ BeginNode.new(load_optional_location, load_optional_node, load_optional_node, load_optional_node, load_optional_node, load_optional_location, location)
150
+ when 11 then
151
+ BlockArgumentNode.new(load_optional_node, load_location, location)
152
+ when 12 then
153
+ BlockNode.new(Array.new(load_varint) { load_constant }, load_optional_node, load_optional_node, load_location, load_location, location)
154
+ when 13 then
155
+ BlockParameterNode.new(load_optional_location, load_location, location)
156
+ when 14 then
157
+ BlockParametersNode.new(load_optional_node, Array.new(load_varint) { load_location }, load_optional_location, load_optional_location, location)
158
+ when 15 then
159
+ BreakNode.new(load_optional_node, load_location, location)
160
+ when 16 then
161
+ CallNode.new(load_optional_node, load_optional_location, load_optional_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, load_varint, load_string, location)
162
+ when 17 then
163
+ CallOperatorAndWriteNode.new(load_node, load_location, load_node, location)
164
+ when 18 then
165
+ CallOperatorOrWriteNode.new(load_node, load_node, load_location, location)
166
+ when 19 then
167
+ CallOperatorWriteNode.new(load_node, load_location, load_node, load_constant, location)
168
+ when 20 then
169
+ CapturePatternNode.new(load_node, load_node, load_location, location)
170
+ when 21 then
171
+ CaseNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_location, load_location, location)
172
+ when 22 then
173
+ ClassNode.new(Array.new(load_varint) { load_constant }, load_location, load_node, load_optional_location, load_optional_node, load_optional_node, load_location, location)
174
+ when 23 then
175
+ ClassVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
176
+ when 24 then
177
+ ClassVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
178
+ when 25 then
179
+ ClassVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
180
+ when 26 then
181
+ ClassVariableReadNode.new(location)
182
+ when 27 then
183
+ ClassVariableWriteNode.new(load_location, load_optional_node, load_optional_location, location)
184
+ when 28 then
185
+ ConstantOperatorAndWriteNode.new(load_location, load_location, load_node, location)
186
+ when 29 then
187
+ ConstantOperatorOrWriteNode.new(load_location, load_location, load_node, location)
188
+ when 30 then
189
+ ConstantOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
190
+ when 31 then
191
+ ConstantPathNode.new(load_optional_node, load_node, load_location, location)
192
+ when 32 then
193
+ ConstantPathOperatorAndWriteNode.new(load_node, load_location, load_node, location)
194
+ when 33 then
195
+ ConstantPathOperatorOrWriteNode.new(load_node, load_location, load_node, location)
196
+ when 34 then
197
+ ConstantPathOperatorWriteNode.new(load_node, load_location, load_node, load_constant, location)
198
+ when 35 then
199
+ ConstantPathWriteNode.new(load_node, load_optional_location, load_optional_node, location)
200
+ when 36 then
201
+ ConstantReadNode.new(location)
202
+ when 37 then
203
+ ConstantWriteNode.new(load_location, load_optional_node, load_optional_location, location)
204
+ when 38 then
205
+ load_serialized_length
206
+ DefNode.new(load_location, load_optional_node, load_optional_node, load_optional_node, Array.new(load_varint) { load_constant }, load_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, location)
207
+ when 39 then
208
+ DefinedNode.new(load_optional_location, load_node, load_optional_location, load_location, location)
209
+ when 40 then
210
+ ElseNode.new(load_location, load_optional_node, load_optional_location, location)
211
+ when 41 then
212
+ EmbeddedStatementsNode.new(load_location, load_optional_node, load_location, location)
213
+ when 42 then
214
+ EmbeddedVariableNode.new(load_location, load_node, location)
215
+ when 43 then
216
+ EnsureNode.new(load_location, load_optional_node, load_location, location)
217
+ when 44 then
218
+ FalseNode.new(location)
219
+ when 45 then
220
+ FindPatternNode.new(load_optional_node, load_node, Array.new(load_varint) { load_node }, load_node, load_optional_location, load_optional_location, location)
221
+ when 46 then
222
+ FloatNode.new(location)
223
+ when 47 then
224
+ ForNode.new(load_node, load_node, load_optional_node, load_location, load_location, load_optional_location, load_location, location)
225
+ when 48 then
226
+ ForwardingArgumentsNode.new(location)
227
+ when 49 then
228
+ ForwardingParameterNode.new(location)
229
+ when 50 then
230
+ ForwardingSuperNode.new(load_optional_node, location)
231
+ when 51 then
232
+ GlobalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
233
+ when 52 then
234
+ GlobalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
235
+ when 53 then
236
+ GlobalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
237
+ when 54 then
238
+ GlobalVariableReadNode.new(location)
239
+ when 55 then
240
+ GlobalVariableWriteNode.new(load_location, load_optional_location, load_optional_node, location)
241
+ when 56 then
242
+ HashNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
243
+ when 57 then
244
+ HashPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_location, load_optional_location, location)
245
+ when 58 then
246
+ IfNode.new(load_optional_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
247
+ when 59 then
248
+ ImaginaryNode.new(load_node, location)
249
+ when 60 then
250
+ InNode.new(load_node, load_optional_node, load_location, load_optional_location, location)
251
+ when 61 then
252
+ InstanceVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
253
+ when 62 then
254
+ InstanceVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
255
+ when 63 then
256
+ InstanceVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
257
+ when 64 then
258
+ InstanceVariableReadNode.new(location)
259
+ when 65 then
260
+ InstanceVariableWriteNode.new(load_location, load_optional_node, load_optional_location, location)
261
+ when 66 then
262
+ IntegerNode.new(location)
263
+ when 67 then
264
+ InterpolatedRegularExpressionNode.new(load_location, Array.new(load_varint) { load_node }, load_location, load_varint, location)
265
+ when 68 then
266
+ InterpolatedStringNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
267
+ when 69 then
268
+ InterpolatedSymbolNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
269
+ when 70 then
270
+ InterpolatedXStringNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
271
+ when 71 then
272
+ KeywordHashNode.new(Array.new(load_varint) { load_node }, location)
273
+ when 72 then
274
+ KeywordParameterNode.new(load_location, load_optional_node, location)
275
+ when 73 then
276
+ KeywordRestParameterNode.new(load_location, load_optional_location, location)
277
+ when 74 then
278
+ LambdaNode.new(Array.new(load_varint) { load_constant }, load_location, load_optional_node, load_optional_node, location)
279
+ when 75 then
280
+ LocalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, load_constant, location)
281
+ when 76 then
282
+ LocalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, load_constant, location)
283
+ when 77 then
284
+ LocalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, load_constant, location)
285
+ when 78 then
286
+ LocalVariableReadNode.new(load_constant, load_varint, location)
287
+ when 79 then
288
+ LocalVariableWriteNode.new(load_constant, load_varint, load_optional_node, load_location, load_optional_location, location)
289
+ when 80 then
290
+ MatchPredicateNode.new(load_node, load_node, load_location, location)
291
+ when 81 then
292
+ MatchRequiredNode.new(load_node, load_node, load_location, location)
293
+ when 82 then
294
+ MissingNode.new(location)
295
+ when 83 then
296
+ ModuleNode.new(Array.new(load_varint) { load_constant }, load_location, load_node, load_optional_node, load_location, location)
297
+ when 84 then
298
+ MultiWriteNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_location, load_optional_location, location)
299
+ when 85 then
300
+ NextNode.new(load_optional_node, load_location, location)
301
+ when 86 then
302
+ NilNode.new(location)
303
+ when 87 then
304
+ NoKeywordsParameterNode.new(load_location, load_location, location)
305
+ when 88 then
306
+ NumberedReferenceReadNode.new(location)
307
+ when 89 then
308
+ OptionalParameterNode.new(load_constant, load_location, load_location, load_node, location)
309
+ when 90 then
310
+ OrNode.new(load_node, load_node, load_location, location)
311
+ when 91 then
312
+ ParametersNode.new(Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_node, location)
313
+ when 92 then
314
+ ParenthesesNode.new(load_optional_node, load_location, load_location, location)
315
+ when 93 then
316
+ PinnedExpressionNode.new(load_node, load_location, load_location, load_location, location)
317
+ when 94 then
318
+ PinnedVariableNode.new(load_node, load_location, location)
319
+ when 95 then
320
+ PostExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
321
+ when 96 then
322
+ PreExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
323
+ when 97 then
324
+ ProgramNode.new(Array.new(load_varint) { load_constant }, load_node, location)
325
+ when 98 then
326
+ RangeNode.new(load_optional_node, load_optional_node, load_location, load_varint, location)
327
+ when 99 then
328
+ RationalNode.new(load_node, location)
329
+ when 100 then
330
+ RedoNode.new(location)
331
+ when 101 then
332
+ RegularExpressionNode.new(load_location, load_location, load_location, load_string, load_varint, location)
333
+ when 102 then
334
+ RequiredDestructuredParameterNode.new(Array.new(load_varint) { load_node }, load_location, load_location, location)
335
+ when 103 then
336
+ RequiredParameterNode.new(load_constant, location)
337
+ when 104 then
338
+ RescueModifierNode.new(load_node, load_location, load_node, location)
339
+ when 105 then
340
+ RescueNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_node, load_optional_node, location)
341
+ when 106 then
342
+ RestParameterNode.new(load_location, load_optional_location, location)
343
+ when 107 then
344
+ RetryNode.new(location)
345
+ when 108 then
346
+ ReturnNode.new(load_location, load_optional_node, location)
347
+ when 109 then
348
+ SelfNode.new(location)
349
+ when 110 then
350
+ SingletonClassNode.new(Array.new(load_varint) { load_constant }, load_location, load_location, load_node, load_optional_node, load_location, location)
351
+ when 111 then
352
+ SourceEncodingNode.new(location)
353
+ when 112 then
354
+ SourceFileNode.new(load_string, location)
355
+ when 113 then
356
+ SourceLineNode.new(location)
357
+ when 114 then
358
+ SplatNode.new(load_location, load_optional_node, location)
359
+ when 115 then
360
+ StatementsNode.new(Array.new(load_varint) { load_node }, location)
361
+ when 116 then
362
+ StringConcatNode.new(load_node, load_node, location)
363
+ when 117 then
364
+ StringNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
365
+ when 118 then
366
+ SuperNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, location)
367
+ when 119 then
368
+ SymbolNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
369
+ when 120 then
370
+ TrueNode.new(location)
371
+ when 121 then
372
+ UndefNode.new(Array.new(load_varint) { load_node }, load_location, location)
373
+ when 122 then
374
+ UnlessNode.new(load_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
375
+ when 123 then
376
+ UntilNode.new(load_location, load_node, load_optional_node, load_varint, location)
377
+ when 124 then
378
+ WhenNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_node, location)
379
+ when 125 then
380
+ WhileNode.new(load_location, load_node, load_optional_node, load_varint, location)
381
+ when 126 then
382
+ XStringNode.new(load_location, load_location, load_location, load_string, location)
383
+ when 127 then
384
+ YieldNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, location)
385
+ end
386
+ end
387
+ end
388
+ end
389
+ end