yarp 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +55 -0
  3. data/CONTRIBUTING.md +4 -0
  4. data/{Makefile.in → Makefile} +5 -4
  5. data/README.md +6 -3
  6. data/config.yml +83 -274
  7. data/docs/build_system.md +4 -15
  8. data/docs/building.md +1 -5
  9. data/docs/encoding.md +1 -0
  10. data/docs/{extension.md → ruby_api.md} +6 -3
  11. data/docs/serialization.md +71 -24
  12. data/ext/yarp/api_node.c +173 -585
  13. data/ext/yarp/extconf.rb +15 -10
  14. data/ext/yarp/extension.c +4 -2
  15. data/ext/yarp/extension.h +1 -1
  16. data/include/yarp/ast.h +167 -306
  17. data/include/yarp/defines.h +5 -15
  18. data/include/yarp/enc/yp_encoding.h +1 -0
  19. data/include/yarp/unescape.h +1 -1
  20. data/include/yarp/util/yp_buffer.h +9 -0
  21. data/include/yarp/util/yp_constant_pool.h +3 -0
  22. data/include/yarp/util/yp_list.h +7 -7
  23. data/include/yarp/util/yp_newline_list.h +4 -0
  24. data/include/yarp/util/yp_state_stack.h +1 -1
  25. data/include/yarp/util/yp_string.h +5 -1
  26. data/include/yarp/version.h +2 -3
  27. data/include/yarp.h +4 -2
  28. data/lib/yarp/ffi.rb +226 -0
  29. data/lib/yarp/lex_compat.rb +16 -2
  30. data/lib/yarp/node.rb +594 -1437
  31. data/lib/yarp/ripper_compat.rb +3 -3
  32. data/lib/yarp/serialize.rb +312 -149
  33. data/lib/yarp.rb +167 -2
  34. data/src/enc/yp_unicode.c +9 -0
  35. data/src/node.c +92 -250
  36. data/src/prettyprint.c +81 -206
  37. data/src/serialize.c +124 -149
  38. data/src/unescape.c +29 -35
  39. data/src/util/yp_buffer.c +18 -0
  40. data/src/util/yp_list.c +7 -16
  41. data/src/util/yp_state_stack.c +0 -6
  42. data/src/util/yp_string.c +8 -17
  43. data/src/yarp.c +444 -717
  44. data/yarp.gemspec +5 -5
  45. metadata +6 -6
  46. data/config.h.in +0 -25
  47. data/configure +0 -4487
data/src/serialize.c CHANGED
@@ -5,10 +5,7 @@
5
5
  /* if you are looking to modify the */
6
6
  /* template */
7
7
  /******************************************************************************/
8
- #include "yarp/ast.h"
9
- #include "yarp/diagnostic.h"
10
- #include "yarp/parser.h"
11
- #include "yarp/util/yp_buffer.h"
8
+ #include "yarp.h"
12
9
 
13
10
  #include <stdio.h>
14
11
 
@@ -61,6 +58,12 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
61
58
  serialize_location(parser, &((yp_and_node_t *)node)->operator_loc, buffer);
62
59
  break;
63
60
  }
61
+ case YP_NODE_AND_WRITE_NODE: {
62
+ yp_serialize_node(parser, (yp_node_t *)((yp_and_write_node_t *)node)->target, buffer);
63
+ yp_serialize_node(parser, (yp_node_t *)((yp_and_write_node_t *)node)->value, buffer);
64
+ serialize_location(parser, &((yp_and_write_node_t *)node)->operator_loc, buffer);
65
+ break;
66
+ }
64
67
  case YP_NODE_ARGUMENTS_NODE: {
65
68
  uint32_t arguments_size = yp_sizet_to_u32(((yp_arguments_node_t *)node)->arguments.size);
66
69
  yp_buffer_append_u32(buffer, arguments_size);
@@ -206,10 +209,10 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
206
209
  } else {
207
210
  yp_serialize_node(parser, (yp_node_t *)((yp_block_node_t *)node)->parameters, buffer);
208
211
  }
209
- if (((yp_block_node_t *)node)->statements == NULL) {
212
+ if (((yp_block_node_t *)node)->body == NULL) {
210
213
  yp_buffer_append_u8(buffer, 0);
211
214
  } else {
212
- yp_serialize_node(parser, (yp_node_t *)((yp_block_node_t *)node)->statements, buffer);
215
+ yp_serialize_node(parser, (yp_node_t *)((yp_block_node_t *)node)->body, buffer);
213
216
  }
214
217
  serialize_location(parser, &((yp_block_node_t *)node)->opening_loc, buffer);
215
218
  serialize_location(parser, &((yp_block_node_t *)node)->closing_loc, buffer);
@@ -299,7 +302,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
299
302
  } else {
300
303
  yp_serialize_node(parser, (yp_node_t *)((yp_call_node_t *)node)->block, buffer);
301
304
  }
302
- yp_buffer_append_u32(buffer, ((yp_call_node_t *)node)->flags);
305
+ yp_buffer_append_u32(buffer, node->flags >> 1);
303
306
  uint32_t name_length = yp_sizet_to_u32(yp_string_length(&((yp_call_node_t *)node)->name));
304
307
  yp_buffer_append_u32(buffer, name_length);
305
308
  yp_buffer_append_str(buffer, yp_string_source(&((yp_call_node_t *)node)->name), name_length);
@@ -369,33 +372,14 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
369
372
  } else {
370
373
  yp_serialize_node(parser, (yp_node_t *)((yp_class_node_t *)node)->superclass, buffer);
371
374
  }
372
- if (((yp_class_node_t *)node)->statements == NULL) {
375
+ if (((yp_class_node_t *)node)->body == NULL) {
373
376
  yp_buffer_append_u8(buffer, 0);
374
377
  } else {
375
- yp_serialize_node(parser, (yp_node_t *)((yp_class_node_t *)node)->statements, buffer);
378
+ yp_serialize_node(parser, (yp_node_t *)((yp_class_node_t *)node)->body, buffer);
376
379
  }
377
380
  serialize_location(parser, &((yp_class_node_t *)node)->end_keyword_loc, buffer);
378
381
  break;
379
382
  }
380
- case YP_NODE_CLASS_VARIABLE_OPERATOR_AND_WRITE_NODE: {
381
- serialize_location(parser, &((yp_class_variable_operator_and_write_node_t *)node)->name_loc, buffer);
382
- serialize_location(parser, &((yp_class_variable_operator_and_write_node_t *)node)->operator_loc, buffer);
383
- yp_serialize_node(parser, (yp_node_t *)((yp_class_variable_operator_and_write_node_t *)node)->value, buffer);
384
- break;
385
- }
386
- case YP_NODE_CLASS_VARIABLE_OPERATOR_OR_WRITE_NODE: {
387
- serialize_location(parser, &((yp_class_variable_operator_or_write_node_t *)node)->name_loc, buffer);
388
- serialize_location(parser, &((yp_class_variable_operator_or_write_node_t *)node)->operator_loc, buffer);
389
- yp_serialize_node(parser, (yp_node_t *)((yp_class_variable_operator_or_write_node_t *)node)->value, buffer);
390
- break;
391
- }
392
- case YP_NODE_CLASS_VARIABLE_OPERATOR_WRITE_NODE: {
393
- serialize_location(parser, &((yp_class_variable_operator_write_node_t *)node)->name_loc, buffer);
394
- serialize_location(parser, &((yp_class_variable_operator_write_node_t *)node)->operator_loc, buffer);
395
- yp_serialize_node(parser, (yp_node_t *)((yp_class_variable_operator_write_node_t *)node)->value, buffer);
396
- yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_class_variable_operator_write_node_t *)node)->operator));
397
- break;
398
- }
399
383
  case YP_NODE_CLASS_VARIABLE_READ_NODE: {
400
384
  break;
401
385
  }
@@ -414,25 +398,6 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
414
398
  }
415
399
  break;
416
400
  }
417
- case YP_NODE_CONSTANT_OPERATOR_AND_WRITE_NODE: {
418
- serialize_location(parser, &((yp_constant_operator_and_write_node_t *)node)->name_loc, buffer);
419
- serialize_location(parser, &((yp_constant_operator_and_write_node_t *)node)->operator_loc, buffer);
420
- yp_serialize_node(parser, (yp_node_t *)((yp_constant_operator_and_write_node_t *)node)->value, buffer);
421
- break;
422
- }
423
- case YP_NODE_CONSTANT_OPERATOR_OR_WRITE_NODE: {
424
- serialize_location(parser, &((yp_constant_operator_or_write_node_t *)node)->name_loc, buffer);
425
- serialize_location(parser, &((yp_constant_operator_or_write_node_t *)node)->operator_loc, buffer);
426
- yp_serialize_node(parser, (yp_node_t *)((yp_constant_operator_or_write_node_t *)node)->value, buffer);
427
- break;
428
- }
429
- case YP_NODE_CONSTANT_OPERATOR_WRITE_NODE: {
430
- serialize_location(parser, &((yp_constant_operator_write_node_t *)node)->name_loc, buffer);
431
- serialize_location(parser, &((yp_constant_operator_write_node_t *)node)->operator_loc, buffer);
432
- yp_serialize_node(parser, (yp_node_t *)((yp_constant_operator_write_node_t *)node)->value, buffer);
433
- yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_constant_operator_write_node_t *)node)->operator));
434
- break;
435
- }
436
401
  case YP_NODE_CONSTANT_PATH_NODE: {
437
402
  if (((yp_constant_path_node_t *)node)->parent == NULL) {
438
403
  yp_buffer_append_u8(buffer, 0);
@@ -443,25 +408,6 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
443
408
  serialize_location(parser, &((yp_constant_path_node_t *)node)->delimiter_loc, buffer);
444
409
  break;
445
410
  }
446
- case YP_NODE_CONSTANT_PATH_OPERATOR_AND_WRITE_NODE: {
447
- yp_serialize_node(parser, (yp_node_t *)((yp_constant_path_operator_and_write_node_t *)node)->target, buffer);
448
- serialize_location(parser, &((yp_constant_path_operator_and_write_node_t *)node)->operator_loc, buffer);
449
- yp_serialize_node(parser, (yp_node_t *)((yp_constant_path_operator_and_write_node_t *)node)->value, buffer);
450
- break;
451
- }
452
- case YP_NODE_CONSTANT_PATH_OPERATOR_OR_WRITE_NODE: {
453
- yp_serialize_node(parser, (yp_node_t *)((yp_constant_path_operator_or_write_node_t *)node)->target, buffer);
454
- serialize_location(parser, &((yp_constant_path_operator_or_write_node_t *)node)->operator_loc, buffer);
455
- yp_serialize_node(parser, (yp_node_t *)((yp_constant_path_operator_or_write_node_t *)node)->value, buffer);
456
- break;
457
- }
458
- case YP_NODE_CONSTANT_PATH_OPERATOR_WRITE_NODE: {
459
- yp_serialize_node(parser, (yp_node_t *)((yp_constant_path_operator_write_node_t *)node)->target, buffer);
460
- serialize_location(parser, &((yp_constant_path_operator_write_node_t *)node)->operator_loc, buffer);
461
- yp_serialize_node(parser, (yp_node_t *)((yp_constant_path_operator_write_node_t *)node)->value, buffer);
462
- yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_constant_path_operator_write_node_t *)node)->operator));
463
- break;
464
- }
465
411
  case YP_NODE_CONSTANT_PATH_WRITE_NODE: {
466
412
  yp_serialize_node(parser, (yp_node_t *)((yp_constant_path_write_node_t *)node)->target, buffer);
467
413
  if (((yp_constant_path_write_node_t *)node)->operator_loc.start == NULL) {
@@ -496,10 +442,10 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
496
442
  break;
497
443
  }
498
444
  case YP_NODE_DEF_NODE: {
499
- // serialize length
500
- // encoding of location u32s make us need to save this offset.
501
- size_t length_offset = buffer->length;
502
- yp_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
445
+ // serialize length
446
+ // encoding of location u32s make us need to save this offset.
447
+ size_t length_offset = buffer->length;
448
+ yp_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
503
449
  serialize_location(parser, &((yp_def_node_t *)node)->name_loc, buffer);
504
450
  if (((yp_def_node_t *)node)->receiver == NULL) {
505
451
  yp_buffer_append_u8(buffer, 0);
@@ -511,10 +457,10 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
511
457
  } else {
512
458
  yp_serialize_node(parser, (yp_node_t *)((yp_def_node_t *)node)->parameters, buffer);
513
459
  }
514
- if (((yp_def_node_t *)node)->statements == NULL) {
460
+ if (((yp_def_node_t *)node)->body == NULL) {
515
461
  yp_buffer_append_u8(buffer, 0);
516
462
  } else {
517
- yp_serialize_node(parser, (yp_node_t *)((yp_def_node_t *)node)->statements, buffer);
463
+ yp_serialize_node(parser, (yp_node_t *)((yp_def_node_t *)node)->body, buffer);
518
464
  }
519
465
  uint32_t locals_size = yp_sizet_to_u32(((yp_def_node_t *)node)->locals.size);
520
466
  yp_buffer_append_u32(buffer, locals_size);
@@ -552,9 +498,9 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
552
498
  yp_buffer_append_u8(buffer, 1);
553
499
  serialize_location(parser, &((yp_def_node_t *)node)->end_keyword_loc, buffer);
554
500
  }
555
- // serialize length
556
- uint32_t length = yp_sizet_to_u32(buffer->length - offset - sizeof(uint32_t));
557
- memcpy(buffer->value + length_offset, &length, sizeof(uint32_t));
501
+ // serialize length
502
+ uint32_t length = yp_sizet_to_u32(buffer->length - offset - sizeof(uint32_t));
503
+ memcpy(buffer->value + length_offset, &length, sizeof(uint32_t));
558
504
  break;
559
505
  }
560
506
  case YP_NODE_DEFINED_NODE: {
@@ -644,6 +590,21 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
644
590
  }
645
591
  break;
646
592
  }
593
+ case YP_NODE_FLIP_FLOP_NODE: {
594
+ if (((yp_flip_flop_node_t *)node)->left == NULL) {
595
+ yp_buffer_append_u8(buffer, 0);
596
+ } else {
597
+ yp_serialize_node(parser, (yp_node_t *)((yp_flip_flop_node_t *)node)->left, buffer);
598
+ }
599
+ if (((yp_flip_flop_node_t *)node)->right == NULL) {
600
+ yp_buffer_append_u8(buffer, 0);
601
+ } else {
602
+ yp_serialize_node(parser, (yp_node_t *)((yp_flip_flop_node_t *)node)->right, buffer);
603
+ }
604
+ serialize_location(parser, &((yp_flip_flop_node_t *)node)->operator_loc, buffer);
605
+ yp_buffer_append_u32(buffer, node->flags >> 1);
606
+ break;
607
+ }
647
608
  case YP_NODE_FLOAT_NODE: {
648
609
  break;
649
610
  }
@@ -680,25 +641,6 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
680
641
  }
681
642
  break;
682
643
  }
683
- case YP_NODE_GLOBAL_VARIABLE_OPERATOR_AND_WRITE_NODE: {
684
- serialize_location(parser, &((yp_global_variable_operator_and_write_node_t *)node)->name_loc, buffer);
685
- serialize_location(parser, &((yp_global_variable_operator_and_write_node_t *)node)->operator_loc, buffer);
686
- yp_serialize_node(parser, (yp_node_t *)((yp_global_variable_operator_and_write_node_t *)node)->value, buffer);
687
- break;
688
- }
689
- case YP_NODE_GLOBAL_VARIABLE_OPERATOR_OR_WRITE_NODE: {
690
- serialize_location(parser, &((yp_global_variable_operator_or_write_node_t *)node)->name_loc, buffer);
691
- serialize_location(parser, &((yp_global_variable_operator_or_write_node_t *)node)->operator_loc, buffer);
692
- yp_serialize_node(parser, (yp_node_t *)((yp_global_variable_operator_or_write_node_t *)node)->value, buffer);
693
- break;
694
- }
695
- case YP_NODE_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE: {
696
- serialize_location(parser, &((yp_global_variable_operator_write_node_t *)node)->name_loc, buffer);
697
- serialize_location(parser, &((yp_global_variable_operator_write_node_t *)node)->operator_loc, buffer);
698
- yp_serialize_node(parser, (yp_node_t *)((yp_global_variable_operator_write_node_t *)node)->value, buffer);
699
- yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_global_variable_operator_write_node_t *)node)->operator));
700
- break;
701
- }
702
644
  case YP_NODE_GLOBAL_VARIABLE_READ_NODE: {
703
645
  break;
704
646
  }
@@ -803,25 +745,6 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
803
745
  }
804
746
  break;
805
747
  }
806
- case YP_NODE_INSTANCE_VARIABLE_OPERATOR_AND_WRITE_NODE: {
807
- serialize_location(parser, &((yp_instance_variable_operator_and_write_node_t *)node)->name_loc, buffer);
808
- serialize_location(parser, &((yp_instance_variable_operator_and_write_node_t *)node)->operator_loc, buffer);
809
- yp_serialize_node(parser, (yp_node_t *)((yp_instance_variable_operator_and_write_node_t *)node)->value, buffer);
810
- break;
811
- }
812
- case YP_NODE_INSTANCE_VARIABLE_OPERATOR_OR_WRITE_NODE: {
813
- serialize_location(parser, &((yp_instance_variable_operator_or_write_node_t *)node)->name_loc, buffer);
814
- serialize_location(parser, &((yp_instance_variable_operator_or_write_node_t *)node)->operator_loc, buffer);
815
- yp_serialize_node(parser, (yp_node_t *)((yp_instance_variable_operator_or_write_node_t *)node)->value, buffer);
816
- break;
817
- }
818
- case YP_NODE_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE: {
819
- serialize_location(parser, &((yp_instance_variable_operator_write_node_t *)node)->name_loc, buffer);
820
- serialize_location(parser, &((yp_instance_variable_operator_write_node_t *)node)->operator_loc, buffer);
821
- yp_serialize_node(parser, (yp_node_t *)((yp_instance_variable_operator_write_node_t *)node)->value, buffer);
822
- yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_instance_variable_operator_write_node_t *)node)->operator));
823
- break;
824
- }
825
748
  case YP_NODE_INSTANCE_VARIABLE_READ_NODE: {
826
749
  break;
827
750
  }
@@ -851,7 +774,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
851
774
  yp_serialize_node(parser, (yp_node_t *) ((yp_interpolated_regular_expression_node_t *)node)->parts.nodes[index], buffer);
852
775
  }
853
776
  serialize_location(parser, &((yp_interpolated_regular_expression_node_t *)node)->closing_loc, buffer);
854
- yp_buffer_append_u32(buffer, ((yp_interpolated_regular_expression_node_t *)node)->flags);
777
+ yp_buffer_append_u32(buffer, node->flags >> 1);
855
778
  break;
856
779
  }
857
780
  case YP_NODE_INTERPOLATED_STRING_NODE: {
@@ -943,35 +866,13 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
943
866
  } else {
944
867
  yp_serialize_node(parser, (yp_node_t *)((yp_lambda_node_t *)node)->parameters, buffer);
945
868
  }
946
- if (((yp_lambda_node_t *)node)->statements == NULL) {
869
+ if (((yp_lambda_node_t *)node)->body == NULL) {
947
870
  yp_buffer_append_u8(buffer, 0);
948
871
  } else {
949
- yp_serialize_node(parser, (yp_node_t *)((yp_lambda_node_t *)node)->statements, buffer);
872
+ yp_serialize_node(parser, (yp_node_t *)((yp_lambda_node_t *)node)->body, buffer);
950
873
  }
951
874
  break;
952
875
  }
953
- case YP_NODE_LOCAL_VARIABLE_OPERATOR_AND_WRITE_NODE: {
954
- serialize_location(parser, &((yp_local_variable_operator_and_write_node_t *)node)->name_loc, buffer);
955
- serialize_location(parser, &((yp_local_variable_operator_and_write_node_t *)node)->operator_loc, buffer);
956
- yp_serialize_node(parser, (yp_node_t *)((yp_local_variable_operator_and_write_node_t *)node)->value, buffer);
957
- yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_local_variable_operator_and_write_node_t *)node)->constant_id));
958
- break;
959
- }
960
- case YP_NODE_LOCAL_VARIABLE_OPERATOR_OR_WRITE_NODE: {
961
- serialize_location(parser, &((yp_local_variable_operator_or_write_node_t *)node)->name_loc, buffer);
962
- serialize_location(parser, &((yp_local_variable_operator_or_write_node_t *)node)->operator_loc, buffer);
963
- yp_serialize_node(parser, (yp_node_t *)((yp_local_variable_operator_or_write_node_t *)node)->value, buffer);
964
- yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_local_variable_operator_or_write_node_t *)node)->constant_id));
965
- break;
966
- }
967
- case YP_NODE_LOCAL_VARIABLE_OPERATOR_WRITE_NODE: {
968
- serialize_location(parser, &((yp_local_variable_operator_write_node_t *)node)->name_loc, buffer);
969
- serialize_location(parser, &((yp_local_variable_operator_write_node_t *)node)->operator_loc, buffer);
970
- yp_serialize_node(parser, (yp_node_t *)((yp_local_variable_operator_write_node_t *)node)->value, buffer);
971
- yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_local_variable_operator_write_node_t *)node)->constant_id));
972
- yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_local_variable_operator_write_node_t *)node)->operator_id));
973
- break;
974
- }
975
876
  case YP_NODE_LOCAL_VARIABLE_READ_NODE: {
976
877
  yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_local_variable_read_node_t *)node)->constant_id));
977
878
  yp_buffer_append_u32(buffer, ((yp_local_variable_read_node_t *)node)->depth);
@@ -1017,10 +918,10 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1017
918
  }
1018
919
  serialize_location(parser, &((yp_module_node_t *)node)->module_keyword_loc, buffer);
1019
920
  yp_serialize_node(parser, (yp_node_t *)((yp_module_node_t *)node)->constant_path, buffer);
1020
- if (((yp_module_node_t *)node)->statements == NULL) {
921
+ if (((yp_module_node_t *)node)->body == NULL) {
1021
922
  yp_buffer_append_u8(buffer, 0);
1022
923
  } else {
1023
- yp_serialize_node(parser, (yp_node_t *)((yp_module_node_t *)node)->statements, buffer);
924
+ yp_serialize_node(parser, (yp_node_t *)((yp_module_node_t *)node)->body, buffer);
1024
925
  }
1025
926
  serialize_location(parser, &((yp_module_node_t *)node)->end_keyword_loc, buffer);
1026
927
  break;
@@ -1076,6 +977,13 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1076
977
  case YP_NODE_NUMBERED_REFERENCE_READ_NODE: {
1077
978
  break;
1078
979
  }
980
+ case YP_NODE_OPERATOR_WRITE_NODE: {
981
+ yp_serialize_node(parser, (yp_node_t *)((yp_operator_write_node_t *)node)->target, buffer);
982
+ serialize_location(parser, &((yp_operator_write_node_t *)node)->operator_loc, buffer);
983
+ yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_operator_write_node_t *)node)->operator));
984
+ yp_serialize_node(parser, (yp_node_t *)((yp_operator_write_node_t *)node)->value, buffer);
985
+ break;
986
+ }
1079
987
  case YP_NODE_OPTIONAL_PARAMETER_NODE: {
1080
988
  yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_optional_parameter_node_t *)node)->constant_id));
1081
989
  serialize_location(parser, &((yp_optional_parameter_node_t *)node)->name_loc, buffer);
@@ -1089,6 +997,12 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1089
997
  serialize_location(parser, &((yp_or_node_t *)node)->operator_loc, buffer);
1090
998
  break;
1091
999
  }
1000
+ case YP_NODE_OR_WRITE_NODE: {
1001
+ yp_serialize_node(parser, (yp_node_t *)((yp_or_write_node_t *)node)->target, buffer);
1002
+ yp_serialize_node(parser, (yp_node_t *)((yp_or_write_node_t *)node)->value, buffer);
1003
+ serialize_location(parser, &((yp_or_write_node_t *)node)->operator_loc, buffer);
1004
+ break;
1005
+ }
1092
1006
  case YP_NODE_PARAMETERS_NODE: {
1093
1007
  uint32_t requireds_size = yp_sizet_to_u32(((yp_parameters_node_t *)node)->requireds.size);
1094
1008
  yp_buffer_append_u32(buffer, requireds_size);
@@ -1128,10 +1042,10 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1128
1042
  break;
1129
1043
  }
1130
1044
  case YP_NODE_PARENTHESES_NODE: {
1131
- if (((yp_parentheses_node_t *)node)->statements == NULL) {
1045
+ if (((yp_parentheses_node_t *)node)->body == NULL) {
1132
1046
  yp_buffer_append_u8(buffer, 0);
1133
1047
  } else {
1134
- yp_serialize_node(parser, (yp_node_t *)((yp_parentheses_node_t *)node)->statements, buffer);
1048
+ yp_serialize_node(parser, (yp_node_t *)((yp_parentheses_node_t *)node)->body, buffer);
1135
1049
  }
1136
1050
  serialize_location(parser, &((yp_parentheses_node_t *)node)->opening_loc, buffer);
1137
1051
  serialize_location(parser, &((yp_parentheses_node_t *)node)->closing_loc, buffer);
@@ -1192,7 +1106,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1192
1106
  yp_serialize_node(parser, (yp_node_t *)((yp_range_node_t *)node)->right, buffer);
1193
1107
  }
1194
1108
  serialize_location(parser, &((yp_range_node_t *)node)->operator_loc, buffer);
1195
- yp_buffer_append_u32(buffer, ((yp_range_node_t *)node)->flags);
1109
+ yp_buffer_append_u32(buffer, node->flags >> 1);
1196
1110
  break;
1197
1111
  }
1198
1112
  case YP_NODE_RATIONAL_NODE: {
@@ -1209,7 +1123,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1209
1123
  uint32_t unescaped_length = yp_sizet_to_u32(yp_string_length(&((yp_regular_expression_node_t *)node)->unescaped));
1210
1124
  yp_buffer_append_u32(buffer, unescaped_length);
1211
1125
  yp_buffer_append_str(buffer, yp_string_source(&((yp_regular_expression_node_t *)node)->unescaped), unescaped_length);
1212
- yp_buffer_append_u32(buffer, ((yp_regular_expression_node_t *)node)->flags);
1126
+ yp_buffer_append_u32(buffer, node->flags >> 1);
1213
1127
  break;
1214
1128
  }
1215
1129
  case YP_NODE_REQUIRED_DESTRUCTURED_PARAMETER_NODE: {
@@ -1296,10 +1210,10 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1296
1210
  serialize_location(parser, &((yp_singleton_class_node_t *)node)->class_keyword_loc, buffer);
1297
1211
  serialize_location(parser, &((yp_singleton_class_node_t *)node)->operator_loc, buffer);
1298
1212
  yp_serialize_node(parser, (yp_node_t *)((yp_singleton_class_node_t *)node)->expression, buffer);
1299
- if (((yp_singleton_class_node_t *)node)->statements == NULL) {
1213
+ if (((yp_singleton_class_node_t *)node)->body == NULL) {
1300
1214
  yp_buffer_append_u8(buffer, 0);
1301
1215
  } else {
1302
- yp_serialize_node(parser, (yp_node_t *)((yp_singleton_class_node_t *)node)->statements, buffer);
1216
+ yp_serialize_node(parser, (yp_node_t *)((yp_singleton_class_node_t *)node)->body, buffer);
1303
1217
  }
1304
1218
  serialize_location(parser, &((yp_singleton_class_node_t *)node)->end_keyword_loc, buffer);
1305
1219
  break;
@@ -1443,7 +1357,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1443
1357
  } else {
1444
1358
  yp_serialize_node(parser, (yp_node_t *)((yp_until_node_t *)node)->statements, buffer);
1445
1359
  }
1446
- yp_buffer_append_u32(buffer, ((yp_until_node_t *)node)->flags);
1360
+ yp_buffer_append_u32(buffer, node->flags >> 1);
1447
1361
  break;
1448
1362
  }
1449
1363
  case YP_NODE_WHEN_NODE: {
@@ -1468,7 +1382,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1468
1382
  } else {
1469
1383
  yp_serialize_node(parser, (yp_node_t *)((yp_while_node_t *)node)->statements, buffer);
1470
1384
  }
1471
- yp_buffer_append_u32(buffer, ((yp_while_node_t *)node)->flags);
1385
+ yp_buffer_append_u32(buffer, node->flags >> 1);
1472
1386
  break;
1473
1387
  }
1474
1388
  case YP_NODE_X_STRING_NODE: {
@@ -1504,6 +1418,24 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1504
1418
  }
1505
1419
  }
1506
1420
 
1421
+ void yp_serialize_comment(yp_parser_t *parser, yp_comment_t *comment, yp_buffer_t *buffer) {
1422
+ // serialize type
1423
+ yp_buffer_append_u8(buffer, (uint8_t) comment->type);
1424
+
1425
+ // serialize location
1426
+ yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->start - parser->start));
1427
+ yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->end - comment->start));
1428
+ }
1429
+
1430
+ void yp_serialize_comment_list(yp_parser_t *parser, yp_list_t list, yp_buffer_t *buffer) {
1431
+ yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(&list)));
1432
+
1433
+ yp_comment_t *comment;
1434
+ for (comment = (yp_comment_t *) list.head; comment != NULL; comment = (yp_comment_t *) comment->node.next) {
1435
+ yp_serialize_comment(parser, comment, buffer);
1436
+ }
1437
+ }
1438
+
1507
1439
  void yp_serialize_diagnostic(yp_parser_t *parser, yp_diagnostic_t *diagnostic, yp_buffer_t *buffer) {
1508
1440
  // serialize message
1509
1441
  size_t message_length = strlen(diagnostic->message);
@@ -1516,7 +1448,7 @@ void yp_serialize_diagnostic(yp_parser_t *parser, yp_diagnostic_t *diagnostic, y
1516
1448
  }
1517
1449
 
1518
1450
  void yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t list, yp_buffer_t *buffer) {
1519
- yp_buffer_append_u32(buffer, yp_list_size(&list));
1451
+ yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(&list)));
1520
1452
 
1521
1453
  yp_diagnostic_t *diagnostic;
1522
1454
  for (diagnostic = (yp_diagnostic_t *) list.head; diagnostic != NULL; diagnostic = (yp_diagnostic_t *) diagnostic->node.next) {
@@ -1524,6 +1456,7 @@ void yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t list, yp_buffer
1524
1456
  }
1525
1457
  }
1526
1458
 
1459
+ #line 145 "serialize.c.erb"
1527
1460
  void
1528
1461
  yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
1529
1462
  // First, serialize the encoding of the parser.
@@ -1531,6 +1464,9 @@ yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer)
1531
1464
  yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length));
1532
1465
  yp_buffer_append_str(buffer, parser->encoding.name, encoding_length);
1533
1466
 
1467
+ // Serialize the comments
1468
+ yp_serialize_comment_list(parser, parser->comment_list, buffer);
1469
+
1534
1470
  // Serialize the errors
1535
1471
  yp_serialize_diagnostic_list(parser, parser->error_list, buffer);
1536
1472
 
@@ -1574,3 +1510,42 @@ yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer)
1574
1510
  }
1575
1511
  }
1576
1512
  }
1513
+
1514
+ static void
1515
+ serialize_token(void *data, yp_parser_t *parser, yp_token_t *token) {
1516
+ yp_buffer_t *buffer = (yp_buffer_t *) data;
1517
+
1518
+ yp_buffer_append_u32(buffer, token->type);
1519
+ yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->start - parser->start));
1520
+ yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->end - token->start));
1521
+ yp_buffer_append_u32(buffer, parser->lex_state);
1522
+ }
1523
+
1524
+ YP_EXPORTED_FUNCTION void
1525
+ yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer) {
1526
+ yp_parser_t parser;
1527
+ yp_parser_init(&parser, source, size, filepath);
1528
+
1529
+ yp_lex_callback_t lex_callback = (yp_lex_callback_t) {
1530
+ .data = (void *) buffer,
1531
+ .callback = serialize_token,
1532
+ };
1533
+
1534
+ parser.lex_callback = &lex_callback;
1535
+ yp_node_t *node = yp_parse(&parser);
1536
+
1537
+ // Append 0 to mark end of tokens
1538
+ yp_buffer_append_u32(buffer, 0);
1539
+
1540
+ // Serialize the comments
1541
+ yp_serialize_comment_list(&parser, parser.comment_list, buffer);
1542
+
1543
+ // Serialize the errors
1544
+ yp_serialize_diagnostic_list(&parser, parser.error_list, buffer);
1545
+
1546
+ // Serialize the warnings
1547
+ yp_serialize_diagnostic_list(&parser, parser.warning_list, buffer);
1548
+
1549
+ yp_node_destroy(&parser, node);
1550
+ yp_parser_free(&parser);
1551
+ }
data/src/unescape.c CHANGED
@@ -180,22 +180,6 @@ unescape_char(const unsigned char value, const unsigned char flags) {
180
180
  static const char *
181
181
  unescape(char *dest, size_t *dest_length, const char *backslash, const char *end, yp_list_t *error_list, const unsigned char flags, bool write_to_str) {
182
182
  switch (backslash[1]) {
183
- // \a \b \e \f \n \r \s \t \v
184
- case '\r': {
185
- // if this is an \r\n we need to escape both
186
- if (write_to_str) {
187
- dest[(*dest_length)++] = (char) unescape_char(unescape_chars[(unsigned char) backslash[1]], flags);
188
- }
189
-
190
- if (backslash + 2 < end && backslash[2] == '\n') {
191
- if (write_to_str) {
192
- dest[(*dest_length)++] = (char) unescape_char(unescape_chars[(unsigned char) backslash[2]], flags);
193
- }
194
- return backslash + 3;
195
- }
196
-
197
- return backslash + 2;
198
- }
199
183
  case 'a':
200
184
  case 'b':
201
185
  case 'e':
@@ -398,14 +382,23 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end
398
382
  yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid meta escape sequence");
399
383
  return backslash + 3;
400
384
  }
385
+ // \n
386
+ case '\n':
387
+ return backslash + 2;
388
+ // \r
389
+ case '\r':
390
+ if (backslash + 2 < end && backslash[2] == '\n') {
391
+ return backslash + 3;
392
+ }
393
+
394
+ /* fallthrough */
401
395
  // In this case we're escaping something that doesn't need escaping.
402
- default:
403
- {
404
- if (write_to_str) {
405
- dest[(*dest_length)++] = backslash[1];
406
- }
407
- return backslash + 2;
396
+ default: {
397
+ if (write_to_str) {
398
+ dest[(*dest_length)++] = backslash[1];
408
399
  }
400
+ return backslash + 2;
401
+ }
409
402
  }
410
403
  }
411
404
 
@@ -438,26 +431,24 @@ unescape(char *dest, size_t *dest_length, const char *backslash, const char *end
438
431
  // \c? or \C-? delete, ASCII 7Fh (DEL)
439
432
  //
440
433
  YP_EXPORTED_FUNCTION void
441
- yp_unescape_manipulate_string(yp_parser_t *parser, const char *value, size_t length, yp_string_t *string, yp_unescape_type_t unescape_type, yp_list_t *error_list) {
434
+ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type, yp_list_t *error_list) {
442
435
  if (unescape_type == YP_UNESCAPE_NONE) {
443
436
  // If we're not unescaping then we can reference the source directly.
444
- yp_string_shared_init(string, value, value + length);
445
437
  return;
446
438
  }
447
439
 
448
- const char *backslash = yp_memchr(value, '\\', length, parser->encoding_changed, &parser->encoding);
440
+ const char *backslash = yp_memchr(string->source, '\\', string->length, parser->encoding_changed, &parser->encoding);
449
441
 
450
442
  if (backslash == NULL) {
451
443
  // Here there are no escapes, so we can reference the source directly.
452
- yp_string_shared_init(string, value, value + length);
453
444
  return;
454
445
  }
455
446
 
456
447
  // Here we have found an escape character, so we need to handle all escapes
457
448
  // within the string.
458
- char *allocated = malloc(length);
449
+ char *allocated = malloc(string->length);
459
450
  if (allocated == NULL) {
460
- yp_diagnostic_list_append(error_list, value, value + length, "Failed to allocate memory for unescaping.");
451
+ yp_diagnostic_list_append(error_list, string->source, string->source + string->length, "Failed to allocate memory for unescaping.");
461
452
  return;
462
453
  }
463
454
 
@@ -468,13 +459,13 @@ yp_unescape_manipulate_string(yp_parser_t *parser, const char *value, size_t len
468
459
  // This is the current position in the source string that we're looking at.
469
460
  // It's going to move along behind the backslash so that we can copy each
470
461
  // segment of the string that doesn't contain an escape.
471
- const char *cursor = value;
472
- const char *end = value + length;
462
+ const char *cursor = string->source;
463
+ const char *end = string->source + string->length;
473
464
 
474
465
  // For each escape found in the source string, we will handle it and update
475
466
  // the moving cursor->backslash window.
476
467
  while (backslash != NULL && backslash + 1 < end) {
477
- assert(dest_length < length);
468
+ assert(dest_length < string->length);
478
469
 
479
470
  // This is the size of the segment of the string from the previous escape
480
471
  // or the start of the string to the current escape.
@@ -520,6 +511,10 @@ yp_unescape_manipulate_string(yp_parser_t *parser, const char *value, size_t len
520
511
  cursor = end;
521
512
  }
522
513
 
514
+ // If the string was already allocated, then we need to free that memory
515
+ // here. That's because we're about to override it with the escaped string.
516
+ yp_string_free(string);
517
+
523
518
  // We also need to update the length at the end. This is because every escape
524
519
  // reduces the length of the final string, and we don't want garbage at the
525
520
  // end.
@@ -530,13 +525,12 @@ YP_EXPORTED_FUNCTION bool
530
525
  yp_unescape_string(const char *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result) {
531
526
  bool success;
532
527
 
533
- yp_list_t error_list;
534
- yp_list_init(&error_list);
535
-
536
528
  yp_parser_t parser;
537
529
  yp_parser_init(&parser, start, length, "");
538
530
 
539
- yp_unescape_manipulate_string(&parser, start, length, result, unescape_type, &error_list);
531
+ yp_list_t error_list = YP_LIST_EMPTY;
532
+ yp_string_shared_init(result, start, start + length);
533
+ yp_unescape_manipulate_string(&parser, result, unescape_type, &error_list);
540
534
  success = yp_list_empty_p(&error_list);
541
535
 
542
536
  yp_list_free(&error_list);
data/src/util/yp_buffer.c CHANGED
@@ -2,6 +2,12 @@
2
2
 
3
3
  #define YP_BUFFER_INITIAL_SIZE 1024
4
4
 
5
+ // Return the size of the yp_buffer_t struct.
6
+ size_t
7
+ yp_buffer_sizeof(void) {
8
+ return sizeof(yp_buffer_t);
9
+ }
10
+
5
11
  // Initialize a yp_buffer_t with its default values.
6
12
  bool
7
13
  yp_buffer_init(yp_buffer_t *buffer) {
@@ -12,6 +18,18 @@ yp_buffer_init(yp_buffer_t *buffer) {
12
18
  return buffer->value != NULL;
13
19
  }
14
20
 
21
+ // Return the value of the buffer.
22
+ char *
23
+ yp_buffer_value(yp_buffer_t *buffer) {
24
+ return buffer->value;
25
+ }
26
+
27
+ // Return the length of the buffer.
28
+ size_t
29
+ yp_buffer_length(yp_buffer_t *buffer) {
30
+ return buffer->length;
31
+ }
32
+
15
33
  // Append the given amount of space to the buffer.
16
34
  static inline void
17
35
  yp_buffer_append_length(yp_buffer_t *buffer, size_t length) {