prism 0.16.0 → 0.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -1
  3. data/Makefile +6 -0
  4. data/README.md +1 -1
  5. data/config.yml +50 -35
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/serialization.md +28 -29
  8. data/ext/prism/api_node.c +802 -770
  9. data/ext/prism/api_pack.c +20 -9
  10. data/ext/prism/extension.c +464 -162
  11. data/ext/prism/extension.h +1 -1
  12. data/include/prism/ast.h +3173 -763
  13. data/include/prism/defines.h +32 -9
  14. data/include/prism/diagnostic.h +36 -3
  15. data/include/prism/enc/pm_encoding.h +118 -28
  16. data/include/prism/node.h +38 -13
  17. data/include/prism/options.h +204 -0
  18. data/include/prism/pack.h +44 -33
  19. data/include/prism/parser.h +445 -200
  20. data/include/prism/prettyprint.h +12 -1
  21. data/include/prism/regexp.h +16 -2
  22. data/include/prism/util/pm_buffer.h +94 -16
  23. data/include/prism/util/pm_char.h +162 -48
  24. data/include/prism/util/pm_constant_pool.h +126 -32
  25. data/include/prism/util/pm_list.h +68 -38
  26. data/include/prism/util/pm_memchr.h +18 -3
  27. data/include/prism/util/pm_newline_list.h +70 -27
  28. data/include/prism/util/pm_state_stack.h +25 -7
  29. data/include/prism/util/pm_string.h +115 -27
  30. data/include/prism/util/pm_string_list.h +25 -6
  31. data/include/prism/util/pm_strncasecmp.h +32 -0
  32. data/include/prism/util/pm_strpbrk.h +31 -17
  33. data/include/prism/version.h +27 -2
  34. data/include/prism.h +224 -31
  35. data/lib/prism/compiler.rb +6 -3
  36. data/lib/prism/debug.rb +23 -7
  37. data/lib/prism/dispatcher.rb +33 -18
  38. data/lib/prism/dsl.rb +10 -5
  39. data/lib/prism/ffi.rb +132 -80
  40. data/lib/prism/lex_compat.rb +25 -15
  41. data/lib/prism/mutation_compiler.rb +10 -5
  42. data/lib/prism/node.rb +370 -135
  43. data/lib/prism/node_ext.rb +1 -1
  44. data/lib/prism/node_inspector.rb +1 -1
  45. data/lib/prism/pack.rb +79 -40
  46. data/lib/prism/parse_result/comments.rb +7 -2
  47. data/lib/prism/parse_result/newlines.rb +4 -0
  48. data/lib/prism/parse_result.rb +150 -30
  49. data/lib/prism/pattern.rb +11 -0
  50. data/lib/prism/ripper_compat.rb +28 -10
  51. data/lib/prism/serialize.rb +86 -54
  52. data/lib/prism/visitor.rb +10 -3
  53. data/lib/prism.rb +20 -2
  54. data/prism.gemspec +4 -2
  55. data/rbi/prism.rbi +104 -60
  56. data/rbi/prism_static.rbi +16 -2
  57. data/sig/prism.rbs +72 -43
  58. data/sig/prism_static.rbs +14 -1
  59. data/src/diagnostic.c +56 -53
  60. data/src/enc/pm_big5.c +1 -0
  61. data/src/enc/pm_euc_jp.c +1 -0
  62. data/src/enc/pm_gbk.c +1 -0
  63. data/src/enc/pm_shift_jis.c +1 -0
  64. data/src/enc/pm_tables.c +316 -80
  65. data/src/enc/pm_unicode.c +53 -8
  66. data/src/enc/pm_windows_31j.c +1 -0
  67. data/src/node.c +334 -321
  68. data/src/options.c +170 -0
  69. data/src/prettyprint.c +74 -47
  70. data/src/prism.c +1642 -856
  71. data/src/regexp.c +151 -95
  72. data/src/serialize.c +44 -20
  73. data/src/token_type.c +3 -1
  74. data/src/util/pm_buffer.c +45 -15
  75. data/src/util/pm_char.c +103 -57
  76. data/src/util/pm_constant_pool.c +51 -21
  77. data/src/util/pm_list.c +12 -4
  78. data/src/util/pm_memchr.c +5 -3
  79. data/src/util/pm_newline_list.c +20 -12
  80. data/src/util/pm_state_stack.c +9 -3
  81. data/src/util/pm_string.c +95 -85
  82. data/src/util/pm_string_list.c +14 -15
  83. data/src/util/pm_strncasecmp.c +10 -3
  84. data/src/util/pm_strpbrk.c +25 -19
  85. metadata +5 -3
  86. data/docs/prism.png +0 -0
data/src/options.c ADDED
@@ -0,0 +1,170 @@
1
+ #include "prism/options.h"
2
+
3
+ /**
4
+ * Set the filepath option on the given options struct.
5
+ */
6
+ PRISM_EXPORTED_FUNCTION void
7
+ pm_options_filepath_set(pm_options_t *options, const char *filepath) {
8
+ pm_string_constant_init(&options->filepath, filepath, strlen(filepath));
9
+ }
10
+
11
+ /**
12
+ * Set the encoding option on the given options struct.
13
+ */
14
+ PRISM_EXPORTED_FUNCTION void
15
+ pm_options_encoding_set(pm_options_t *options, const char *encoding) {
16
+ pm_string_constant_init(&options->encoding, encoding, strlen(encoding));
17
+ }
18
+
19
+ /**
20
+ * Set the line option on the given options struct.
21
+ */
22
+ PRISM_EXPORTED_FUNCTION void
23
+ pm_options_line_set(pm_options_t *options, uint32_t line) {
24
+ options->line = line;
25
+ }
26
+
27
+ /**
28
+ * Set the frozen string literal option on the given options struct.
29
+ */
30
+ PRISM_EXPORTED_FUNCTION void
31
+ pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal) {
32
+ options->frozen_string_literal = frozen_string_literal;
33
+ }
34
+
35
+ /**
36
+ * Set the suppress warnings option on the given options struct.
37
+ */
38
+ PRISM_EXPORTED_FUNCTION void
39
+ pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings) {
40
+ options->suppress_warnings = suppress_warnings;
41
+ }
42
+
43
+ /**
44
+ * Allocate and zero out the scopes array on the given options struct.
45
+ */
46
+ PRISM_EXPORTED_FUNCTION void
47
+ pm_options_scopes_init(pm_options_t *options, size_t scopes_count) {
48
+ options->scopes_count = scopes_count;
49
+ options->scopes = calloc(scopes_count, sizeof(pm_options_scope_t));
50
+ if (options->scopes == NULL) abort();
51
+ }
52
+
53
+ /**
54
+ * Return a pointer to the scope at the given index within the given options.
55
+ */
56
+ PRISM_EXPORTED_FUNCTION const pm_options_scope_t *
57
+ pm_options_scope_get(const pm_options_t *options, size_t index) {
58
+ return &options->scopes[index];
59
+ }
60
+
61
+ /**
62
+ * Create a new options scope struct. This will hold a set of locals that are in
63
+ * scope surrounding the code that is being parsed.
64
+ */
65
+ PRISM_EXPORTED_FUNCTION void
66
+ pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count) {
67
+ scope->locals_count = locals_count;
68
+ scope->locals = calloc(locals_count, sizeof(pm_string_t));
69
+ if (scope->locals == NULL) abort();
70
+ }
71
+
72
+ /**
73
+ * Return a pointer to the local at the given index within the given scope.
74
+ */
75
+ PRISM_EXPORTED_FUNCTION const pm_string_t *
76
+ pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index) {
77
+ return &scope->locals[index];
78
+ }
79
+
80
+ /**
81
+ * Free the internal memory associated with the options.
82
+ */
83
+ PRISM_EXPORTED_FUNCTION void
84
+ pm_options_free(pm_options_t *options) {
85
+ pm_string_free(&options->filepath);
86
+ pm_string_free(&options->encoding);
87
+
88
+ for (size_t scope_index = 0; scope_index < options->scopes_count; scope_index++) {
89
+ pm_options_scope_t *scope = &options->scopes[scope_index];
90
+
91
+ for (size_t local_index = 0; local_index < scope->locals_count; local_index++) {
92
+ pm_string_free(&scope->locals[local_index]);
93
+ }
94
+
95
+ free(scope->locals);
96
+ }
97
+
98
+ free(options->scopes);
99
+ }
100
+
101
+ /**
102
+ * Read a 32-bit unsigned integer from a pointer. This function is used to read
103
+ * the options that are passed into the parser from the Ruby implementation. It
104
+ * handles aligned and unaligned reads.
105
+ */
106
+ static uint32_t
107
+ pm_options_read_u32(const char *data) {
108
+ if (((uintptr_t) data) % sizeof(uint32_t) == 0) {
109
+ return *((uint32_t *) data);
110
+ } else {
111
+ uint32_t value;
112
+ memcpy(&value, data, sizeof(uint32_t));
113
+ return value;
114
+ }
115
+ }
116
+
117
+ /**
118
+ * Deserialize an options struct from the given binary string. This is used to
119
+ * pass options to the parser from an FFI call so that consumers of the library
120
+ * from an FFI perspective don't have to worry about the structure of our
121
+ * options structs. Since the source of these calls will be from Ruby
122
+ * implementation internals we assume it is from a trusted source.
123
+ */
124
+ void
125
+ pm_options_read(pm_options_t *options, const char *data) {
126
+ uint32_t filepath_length = pm_options_read_u32(data);
127
+ data += 4;
128
+
129
+ if (filepath_length > 0) {
130
+ pm_string_constant_init(&options->filepath, data, filepath_length);
131
+ data += filepath_length;
132
+ }
133
+
134
+ options->line = pm_options_read_u32(data);
135
+ data += 4;
136
+
137
+ uint32_t encoding_length = pm_options_read_u32(data);
138
+ data += 4;
139
+
140
+ if (encoding_length > 0) {
141
+ pm_string_constant_init(&options->encoding, data, encoding_length);
142
+ data += encoding_length;
143
+ }
144
+
145
+ options->frozen_string_literal = *data++;
146
+ options->suppress_warnings = *data++;
147
+
148
+ uint32_t scopes_count = pm_options_read_u32(data);
149
+ data += 4;
150
+
151
+ if (scopes_count > 0) {
152
+ pm_options_scopes_init(options, scopes_count);
153
+
154
+ for (size_t scope_index = 0; scope_index < scopes_count; scope_index++) {
155
+ uint32_t locals_count = pm_options_read_u32(data);
156
+ data += 4;
157
+
158
+ pm_options_scope_t *scope = &options->scopes[scope_index];
159
+ pm_options_scope_init(scope, locals_count);
160
+
161
+ for (size_t local_index = 0; local_index < locals_count; local_index++) {
162
+ uint32_t local_length = pm_options_read_u32(data);
163
+ data += 4;
164
+
165
+ pm_string_constant_init(&scope->locals[local_index], data, local_length);
166
+ data += local_length;
167
+ }
168
+ }
169
+ }
170
+ }
data/src/prettyprint.c CHANGED
@@ -5123,52 +5123,6 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
5123
5123
 
5124
5124
  break;
5125
5125
  }
5126
- case PM_KEYWORD_PARAMETER_NODE: {
5127
- pm_keyword_parameter_node_t *cast = (pm_keyword_parameter_node_t *) node;
5128
- pm_buffer_append_string(output_buffer, "@ KeywordParameterNode (location: ", 34);
5129
- prettyprint_location(output_buffer, parser, &node->location);
5130
- pm_buffer_append_string(output_buffer, ")\n", 2);
5131
-
5132
- // name
5133
- {
5134
- pm_buffer_concat(output_buffer, prefix_buffer);
5135
- pm_buffer_append_string(output_buffer, "├── name:", 15);
5136
- pm_buffer_append_byte(output_buffer, ' ');
5137
- prettyprint_constant(output_buffer, parser, cast->name);
5138
- pm_buffer_append_byte(output_buffer, '\n');
5139
- }
5140
-
5141
- // name_loc
5142
- {
5143
- pm_buffer_concat(output_buffer, prefix_buffer);
5144
- pm_buffer_append_string(output_buffer, "├── name_loc:", 19);
5145
- pm_location_t *location = &cast->name_loc;
5146
- pm_buffer_append_byte(output_buffer, ' ');
5147
- prettyprint_location(output_buffer, parser, location);
5148
- pm_buffer_append_string(output_buffer, " = \"", 4);
5149
- prettyprint_source(output_buffer, location->start, (size_t) (location->end - location->start));
5150
- pm_buffer_append_string(output_buffer, "\"\n", 2);
5151
- }
5152
-
5153
- // value
5154
- {
5155
- pm_buffer_concat(output_buffer, prefix_buffer);
5156
- pm_buffer_append_string(output_buffer, "└── value:", 16);
5157
- if (cast->value == NULL) {
5158
- pm_buffer_append_string(output_buffer, " ∅\n", 5);
5159
- } else {
5160
- pm_buffer_append_byte(output_buffer, '\n');
5161
-
5162
- size_t prefix_length = prefix_buffer->length;
5163
- pm_buffer_append_string(prefix_buffer, " ", 4);
5164
- pm_buffer_concat(output_buffer, prefix_buffer);
5165
- prettyprint_node(output_buffer, parser, (pm_node_t *) cast->value, prefix_buffer);
5166
- prefix_buffer->length = prefix_length;
5167
- }
5168
- }
5169
-
5170
- break;
5171
- }
5172
5126
  case PM_KEYWORD_REST_PARAMETER_NODE: {
5173
5127
  pm_keyword_rest_parameter_node_t *cast = (pm_keyword_rest_parameter_node_t *) node;
5174
5128
  pm_buffer_append_string(output_buffer, "@ KeywordRestParameterNode (location: ", 38);
@@ -6256,6 +6210,48 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
6256
6210
 
6257
6211
  break;
6258
6212
  }
6213
+ case PM_OPTIONAL_KEYWORD_PARAMETER_NODE: {
6214
+ pm_optional_keyword_parameter_node_t *cast = (pm_optional_keyword_parameter_node_t *) node;
6215
+ pm_buffer_append_string(output_buffer, "@ OptionalKeywordParameterNode (location: ", 42);
6216
+ prettyprint_location(output_buffer, parser, &node->location);
6217
+ pm_buffer_append_string(output_buffer, ")\n", 2);
6218
+
6219
+ // name
6220
+ {
6221
+ pm_buffer_concat(output_buffer, prefix_buffer);
6222
+ pm_buffer_append_string(output_buffer, "├── name:", 15);
6223
+ pm_buffer_append_byte(output_buffer, ' ');
6224
+ prettyprint_constant(output_buffer, parser, cast->name);
6225
+ pm_buffer_append_byte(output_buffer, '\n');
6226
+ }
6227
+
6228
+ // name_loc
6229
+ {
6230
+ pm_buffer_concat(output_buffer, prefix_buffer);
6231
+ pm_buffer_append_string(output_buffer, "├── name_loc:", 19);
6232
+ pm_location_t *location = &cast->name_loc;
6233
+ pm_buffer_append_byte(output_buffer, ' ');
6234
+ prettyprint_location(output_buffer, parser, location);
6235
+ pm_buffer_append_string(output_buffer, " = \"", 4);
6236
+ prettyprint_source(output_buffer, location->start, (size_t) (location->end - location->start));
6237
+ pm_buffer_append_string(output_buffer, "\"\n", 2);
6238
+ }
6239
+
6240
+ // value
6241
+ {
6242
+ pm_buffer_concat(output_buffer, prefix_buffer);
6243
+ pm_buffer_append_string(output_buffer, "└── value:", 16);
6244
+ pm_buffer_append_byte(output_buffer, '\n');
6245
+
6246
+ size_t prefix_length = prefix_buffer->length;
6247
+ pm_buffer_append_string(prefix_buffer, " ", 4);
6248
+ pm_buffer_concat(output_buffer, prefix_buffer);
6249
+ prettyprint_node(output_buffer, parser, (pm_node_t *) cast->value, prefix_buffer);
6250
+ prefix_buffer->length = prefix_length;
6251
+ }
6252
+
6253
+ break;
6254
+ }
6259
6255
  case PM_OPTIONAL_PARAMETER_NODE: {
6260
6256
  pm_optional_parameter_node_t *cast = (pm_optional_parameter_node_t *) node;
6261
6257
  pm_buffer_append_string(output_buffer, "@ OptionalParameterNode (location: ", 35);
@@ -7007,6 +7003,35 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
7007
7003
 
7008
7004
  break;
7009
7005
  }
7006
+ case PM_REQUIRED_KEYWORD_PARAMETER_NODE: {
7007
+ pm_required_keyword_parameter_node_t *cast = (pm_required_keyword_parameter_node_t *) node;
7008
+ pm_buffer_append_string(output_buffer, "@ RequiredKeywordParameterNode (location: ", 42);
7009
+ prettyprint_location(output_buffer, parser, &node->location);
7010
+ pm_buffer_append_string(output_buffer, ")\n", 2);
7011
+
7012
+ // name
7013
+ {
7014
+ pm_buffer_concat(output_buffer, prefix_buffer);
7015
+ pm_buffer_append_string(output_buffer, "├── name:", 15);
7016
+ pm_buffer_append_byte(output_buffer, ' ');
7017
+ prettyprint_constant(output_buffer, parser, cast->name);
7018
+ pm_buffer_append_byte(output_buffer, '\n');
7019
+ }
7020
+
7021
+ // name_loc
7022
+ {
7023
+ pm_buffer_concat(output_buffer, prefix_buffer);
7024
+ pm_buffer_append_string(output_buffer, "└── name_loc:", 19);
7025
+ pm_location_t *location = &cast->name_loc;
7026
+ pm_buffer_append_byte(output_buffer, ' ');
7027
+ prettyprint_location(output_buffer, parser, location);
7028
+ pm_buffer_append_string(output_buffer, " = \"", 4);
7029
+ prettyprint_source(output_buffer, location->start, (size_t) (location->end - location->start));
7030
+ pm_buffer_append_string(output_buffer, "\"\n", 2);
7031
+ }
7032
+
7033
+ break;
7034
+ }
7010
7035
  case PM_REQUIRED_PARAMETER_NODE: {
7011
7036
  pm_required_parameter_node_t *cast = (pm_required_parameter_node_t *) node;
7012
7037
  pm_buffer_append_string(output_buffer, "@ RequiredParameterNode (location: ", 35);
@@ -8211,7 +8236,9 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
8211
8236
  }
8212
8237
  }
8213
8238
 
8214
- // Pretty-prints the AST represented by the given node to the given buffer.
8239
+ /**
8240
+ * Pretty-prints the AST represented by the given node to the given buffer.
8241
+ */
8215
8242
  PRISM_EXPORTED_FUNCTION void
8216
8243
  pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node) {
8217
8244
  pm_buffer_t prefix_buffer = { 0 };