prism 0.16.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -1
  3. data/Makefile +6 -0
  4. data/README.md +1 -1
  5. data/config.yml +50 -35
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/serialization.md +28 -29
  8. data/ext/prism/api_node.c +802 -770
  9. data/ext/prism/api_pack.c +20 -9
  10. data/ext/prism/extension.c +464 -162
  11. data/ext/prism/extension.h +1 -1
  12. data/include/prism/ast.h +3173 -763
  13. data/include/prism/defines.h +32 -9
  14. data/include/prism/diagnostic.h +36 -3
  15. data/include/prism/enc/pm_encoding.h +118 -28
  16. data/include/prism/node.h +38 -13
  17. data/include/prism/options.h +204 -0
  18. data/include/prism/pack.h +44 -33
  19. data/include/prism/parser.h +445 -200
  20. data/include/prism/prettyprint.h +12 -1
  21. data/include/prism/regexp.h +16 -2
  22. data/include/prism/util/pm_buffer.h +94 -16
  23. data/include/prism/util/pm_char.h +162 -48
  24. data/include/prism/util/pm_constant_pool.h +126 -32
  25. data/include/prism/util/pm_list.h +68 -38
  26. data/include/prism/util/pm_memchr.h +18 -3
  27. data/include/prism/util/pm_newline_list.h +70 -27
  28. data/include/prism/util/pm_state_stack.h +25 -7
  29. data/include/prism/util/pm_string.h +115 -27
  30. data/include/prism/util/pm_string_list.h +25 -6
  31. data/include/prism/util/pm_strncasecmp.h +32 -0
  32. data/include/prism/util/pm_strpbrk.h +31 -17
  33. data/include/prism/version.h +27 -2
  34. data/include/prism.h +224 -31
  35. data/lib/prism/compiler.rb +6 -3
  36. data/lib/prism/debug.rb +23 -7
  37. data/lib/prism/dispatcher.rb +33 -18
  38. data/lib/prism/dsl.rb +10 -5
  39. data/lib/prism/ffi.rb +132 -80
  40. data/lib/prism/lex_compat.rb +25 -15
  41. data/lib/prism/mutation_compiler.rb +10 -5
  42. data/lib/prism/node.rb +370 -135
  43. data/lib/prism/node_ext.rb +1 -1
  44. data/lib/prism/node_inspector.rb +1 -1
  45. data/lib/prism/pack.rb +79 -40
  46. data/lib/prism/parse_result/comments.rb +7 -2
  47. data/lib/prism/parse_result/newlines.rb +4 -0
  48. data/lib/prism/parse_result.rb +150 -30
  49. data/lib/prism/pattern.rb +11 -0
  50. data/lib/prism/ripper_compat.rb +28 -10
  51. data/lib/prism/serialize.rb +86 -54
  52. data/lib/prism/visitor.rb +10 -3
  53. data/lib/prism.rb +20 -2
  54. data/prism.gemspec +4 -2
  55. data/rbi/prism.rbi +104 -60
  56. data/rbi/prism_static.rbi +16 -2
  57. data/sig/prism.rbs +72 -43
  58. data/sig/prism_static.rbs +14 -1
  59. data/src/diagnostic.c +56 -53
  60. data/src/enc/pm_big5.c +1 -0
  61. data/src/enc/pm_euc_jp.c +1 -0
  62. data/src/enc/pm_gbk.c +1 -0
  63. data/src/enc/pm_shift_jis.c +1 -0
  64. data/src/enc/pm_tables.c +316 -80
  65. data/src/enc/pm_unicode.c +53 -8
  66. data/src/enc/pm_windows_31j.c +1 -0
  67. data/src/node.c +334 -321
  68. data/src/options.c +170 -0
  69. data/src/prettyprint.c +74 -47
  70. data/src/prism.c +1642 -856
  71. data/src/regexp.c +151 -95
  72. data/src/serialize.c +44 -20
  73. data/src/token_type.c +3 -1
  74. data/src/util/pm_buffer.c +45 -15
  75. data/src/util/pm_char.c +103 -57
  76. data/src/util/pm_constant_pool.c +51 -21
  77. data/src/util/pm_list.c +12 -4
  78. data/src/util/pm_memchr.c +5 -3
  79. data/src/util/pm_newline_list.c +20 -12
  80. data/src/util/pm_state_stack.c +9 -3
  81. data/src/util/pm_string.c +95 -85
  82. data/src/util/pm_string_list.c +14 -15
  83. data/src/util/pm_strncasecmp.c +10 -3
  84. data/src/util/pm_strpbrk.c +25 -19
  85. metadata +5 -3
  86. data/docs/prism.png +0 -0
data/src/options.c ADDED
@@ -0,0 +1,170 @@
1
+ #include "prism/options.h"
2
+
3
+ /**
4
+ * Set the filepath option on the given options struct.
5
+ */
6
+ PRISM_EXPORTED_FUNCTION void
7
+ pm_options_filepath_set(pm_options_t *options, const char *filepath) {
8
+ pm_string_constant_init(&options->filepath, filepath, strlen(filepath));
9
+ }
10
+
11
+ /**
12
+ * Set the encoding option on the given options struct.
13
+ */
14
+ PRISM_EXPORTED_FUNCTION void
15
+ pm_options_encoding_set(pm_options_t *options, const char *encoding) {
16
+ pm_string_constant_init(&options->encoding, encoding, strlen(encoding));
17
+ }
18
+
19
+ /**
20
+ * Set the line option on the given options struct.
21
+ */
22
+ PRISM_EXPORTED_FUNCTION void
23
+ pm_options_line_set(pm_options_t *options, uint32_t line) {
24
+ options->line = line;
25
+ }
26
+
27
+ /**
28
+ * Set the frozen string literal option on the given options struct.
29
+ */
30
+ PRISM_EXPORTED_FUNCTION void
31
+ pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal) {
32
+ options->frozen_string_literal = frozen_string_literal;
33
+ }
34
+
35
+ /**
36
+ * Set the suppress warnings option on the given options struct.
37
+ */
38
+ PRISM_EXPORTED_FUNCTION void
39
+ pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings) {
40
+ options->suppress_warnings = suppress_warnings;
41
+ }
42
+
43
+ /**
44
+ * Allocate and zero out the scopes array on the given options struct.
45
+ */
46
+ PRISM_EXPORTED_FUNCTION void
47
+ pm_options_scopes_init(pm_options_t *options, size_t scopes_count) {
48
+ options->scopes_count = scopes_count;
49
+ options->scopes = calloc(scopes_count, sizeof(pm_options_scope_t));
50
+ if (options->scopes == NULL) abort();
51
+ }
52
+
53
+ /**
54
+ * Return a pointer to the scope at the given index within the given options.
55
+ */
56
+ PRISM_EXPORTED_FUNCTION const pm_options_scope_t *
57
+ pm_options_scope_get(const pm_options_t *options, size_t index) {
58
+ return &options->scopes[index];
59
+ }
60
+
61
+ /**
62
+ * Create a new options scope struct. This will hold a set of locals that are in
63
+ * scope surrounding the code that is being parsed.
64
+ */
65
+ PRISM_EXPORTED_FUNCTION void
66
+ pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count) {
67
+ scope->locals_count = locals_count;
68
+ scope->locals = calloc(locals_count, sizeof(pm_string_t));
69
+ if (scope->locals == NULL) abort();
70
+ }
71
+
72
+ /**
73
+ * Return a pointer to the local at the given index within the given scope.
74
+ */
75
+ PRISM_EXPORTED_FUNCTION const pm_string_t *
76
+ pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index) {
77
+ return &scope->locals[index];
78
+ }
79
+
80
+ /**
81
+ * Free the internal memory associated with the options.
82
+ */
83
+ PRISM_EXPORTED_FUNCTION void
84
+ pm_options_free(pm_options_t *options) {
85
+ pm_string_free(&options->filepath);
86
+ pm_string_free(&options->encoding);
87
+
88
+ for (size_t scope_index = 0; scope_index < options->scopes_count; scope_index++) {
89
+ pm_options_scope_t *scope = &options->scopes[scope_index];
90
+
91
+ for (size_t local_index = 0; local_index < scope->locals_count; local_index++) {
92
+ pm_string_free(&scope->locals[local_index]);
93
+ }
94
+
95
+ free(scope->locals);
96
+ }
97
+
98
+ free(options->scopes);
99
+ }
100
+
101
+ /**
102
+ * Read a 32-bit unsigned integer from a pointer. This function is used to read
103
+ * the options that are passed into the parser from the Ruby implementation. It
104
+ * handles aligned and unaligned reads.
105
+ */
106
+ static uint32_t
107
+ pm_options_read_u32(const char *data) {
108
+ if (((uintptr_t) data) % sizeof(uint32_t) == 0) {
109
+ return *((uint32_t *) data);
110
+ } else {
111
+ uint32_t value;
112
+ memcpy(&value, data, sizeof(uint32_t));
113
+ return value;
114
+ }
115
+ }
116
+
117
+ /**
118
+ * Deserialize an options struct from the given binary string. This is used to
119
+ * pass options to the parser from an FFI call so that consumers of the library
120
+ * from an FFI perspective don't have to worry about the structure of our
121
+ * options structs. Since the source of these calls will be from Ruby
122
+ * implementation internals we assume it is from a trusted source.
123
+ */
124
+ void
125
+ pm_options_read(pm_options_t *options, const char *data) {
126
+ uint32_t filepath_length = pm_options_read_u32(data);
127
+ data += 4;
128
+
129
+ if (filepath_length > 0) {
130
+ pm_string_constant_init(&options->filepath, data, filepath_length);
131
+ data += filepath_length;
132
+ }
133
+
134
+ options->line = pm_options_read_u32(data);
135
+ data += 4;
136
+
137
+ uint32_t encoding_length = pm_options_read_u32(data);
138
+ data += 4;
139
+
140
+ if (encoding_length > 0) {
141
+ pm_string_constant_init(&options->encoding, data, encoding_length);
142
+ data += encoding_length;
143
+ }
144
+
145
+ options->frozen_string_literal = *data++;
146
+ options->suppress_warnings = *data++;
147
+
148
+ uint32_t scopes_count = pm_options_read_u32(data);
149
+ data += 4;
150
+
151
+ if (scopes_count > 0) {
152
+ pm_options_scopes_init(options, scopes_count);
153
+
154
+ for (size_t scope_index = 0; scope_index < scopes_count; scope_index++) {
155
+ uint32_t locals_count = pm_options_read_u32(data);
156
+ data += 4;
157
+
158
+ pm_options_scope_t *scope = &options->scopes[scope_index];
159
+ pm_options_scope_init(scope, locals_count);
160
+
161
+ for (size_t local_index = 0; local_index < locals_count; local_index++) {
162
+ uint32_t local_length = pm_options_read_u32(data);
163
+ data += 4;
164
+
165
+ pm_string_constant_init(&scope->locals[local_index], data, local_length);
166
+ data += local_length;
167
+ }
168
+ }
169
+ }
170
+ }
data/src/prettyprint.c CHANGED
@@ -5123,52 +5123,6 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
5123
5123
 
5124
5124
  break;
5125
5125
  }
5126
- case PM_KEYWORD_PARAMETER_NODE: {
5127
- pm_keyword_parameter_node_t *cast = (pm_keyword_parameter_node_t *) node;
5128
- pm_buffer_append_string(output_buffer, "@ KeywordParameterNode (location: ", 34);
5129
- prettyprint_location(output_buffer, parser, &node->location);
5130
- pm_buffer_append_string(output_buffer, ")\n", 2);
5131
-
5132
- // name
5133
- {
5134
- pm_buffer_concat(output_buffer, prefix_buffer);
5135
- pm_buffer_append_string(output_buffer, "├── name:", 15);
5136
- pm_buffer_append_byte(output_buffer, ' ');
5137
- prettyprint_constant(output_buffer, parser, cast->name);
5138
- pm_buffer_append_byte(output_buffer, '\n');
5139
- }
5140
-
5141
- // name_loc
5142
- {
5143
- pm_buffer_concat(output_buffer, prefix_buffer);
5144
- pm_buffer_append_string(output_buffer, "├── name_loc:", 19);
5145
- pm_location_t *location = &cast->name_loc;
5146
- pm_buffer_append_byte(output_buffer, ' ');
5147
- prettyprint_location(output_buffer, parser, location);
5148
- pm_buffer_append_string(output_buffer, " = \"", 4);
5149
- prettyprint_source(output_buffer, location->start, (size_t) (location->end - location->start));
5150
- pm_buffer_append_string(output_buffer, "\"\n", 2);
5151
- }
5152
-
5153
- // value
5154
- {
5155
- pm_buffer_concat(output_buffer, prefix_buffer);
5156
- pm_buffer_append_string(output_buffer, "└── value:", 16);
5157
- if (cast->value == NULL) {
5158
- pm_buffer_append_string(output_buffer, " ∅\n", 5);
5159
- } else {
5160
- pm_buffer_append_byte(output_buffer, '\n');
5161
-
5162
- size_t prefix_length = prefix_buffer->length;
5163
- pm_buffer_append_string(prefix_buffer, " ", 4);
5164
- pm_buffer_concat(output_buffer, prefix_buffer);
5165
- prettyprint_node(output_buffer, parser, (pm_node_t *) cast->value, prefix_buffer);
5166
- prefix_buffer->length = prefix_length;
5167
- }
5168
- }
5169
-
5170
- break;
5171
- }
5172
5126
  case PM_KEYWORD_REST_PARAMETER_NODE: {
5173
5127
  pm_keyword_rest_parameter_node_t *cast = (pm_keyword_rest_parameter_node_t *) node;
5174
5128
  pm_buffer_append_string(output_buffer, "@ KeywordRestParameterNode (location: ", 38);
@@ -6256,6 +6210,48 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
6256
6210
 
6257
6211
  break;
6258
6212
  }
6213
+ case PM_OPTIONAL_KEYWORD_PARAMETER_NODE: {
6214
+ pm_optional_keyword_parameter_node_t *cast = (pm_optional_keyword_parameter_node_t *) node;
6215
+ pm_buffer_append_string(output_buffer, "@ OptionalKeywordParameterNode (location: ", 42);
6216
+ prettyprint_location(output_buffer, parser, &node->location);
6217
+ pm_buffer_append_string(output_buffer, ")\n", 2);
6218
+
6219
+ // name
6220
+ {
6221
+ pm_buffer_concat(output_buffer, prefix_buffer);
6222
+ pm_buffer_append_string(output_buffer, "├── name:", 15);
6223
+ pm_buffer_append_byte(output_buffer, ' ');
6224
+ prettyprint_constant(output_buffer, parser, cast->name);
6225
+ pm_buffer_append_byte(output_buffer, '\n');
6226
+ }
6227
+
6228
+ // name_loc
6229
+ {
6230
+ pm_buffer_concat(output_buffer, prefix_buffer);
6231
+ pm_buffer_append_string(output_buffer, "├── name_loc:", 19);
6232
+ pm_location_t *location = &cast->name_loc;
6233
+ pm_buffer_append_byte(output_buffer, ' ');
6234
+ prettyprint_location(output_buffer, parser, location);
6235
+ pm_buffer_append_string(output_buffer, " = \"", 4);
6236
+ prettyprint_source(output_buffer, location->start, (size_t) (location->end - location->start));
6237
+ pm_buffer_append_string(output_buffer, "\"\n", 2);
6238
+ }
6239
+
6240
+ // value
6241
+ {
6242
+ pm_buffer_concat(output_buffer, prefix_buffer);
6243
+ pm_buffer_append_string(output_buffer, "└── value:", 16);
6244
+ pm_buffer_append_byte(output_buffer, '\n');
6245
+
6246
+ size_t prefix_length = prefix_buffer->length;
6247
+ pm_buffer_append_string(prefix_buffer, " ", 4);
6248
+ pm_buffer_concat(output_buffer, prefix_buffer);
6249
+ prettyprint_node(output_buffer, parser, (pm_node_t *) cast->value, prefix_buffer);
6250
+ prefix_buffer->length = prefix_length;
6251
+ }
6252
+
6253
+ break;
6254
+ }
6259
6255
  case PM_OPTIONAL_PARAMETER_NODE: {
6260
6256
  pm_optional_parameter_node_t *cast = (pm_optional_parameter_node_t *) node;
6261
6257
  pm_buffer_append_string(output_buffer, "@ OptionalParameterNode (location: ", 35);
@@ -7007,6 +7003,35 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
7007
7003
 
7008
7004
  break;
7009
7005
  }
7006
+ case PM_REQUIRED_KEYWORD_PARAMETER_NODE: {
7007
+ pm_required_keyword_parameter_node_t *cast = (pm_required_keyword_parameter_node_t *) node;
7008
+ pm_buffer_append_string(output_buffer, "@ RequiredKeywordParameterNode (location: ", 42);
7009
+ prettyprint_location(output_buffer, parser, &node->location);
7010
+ pm_buffer_append_string(output_buffer, ")\n", 2);
7011
+
7012
+ // name
7013
+ {
7014
+ pm_buffer_concat(output_buffer, prefix_buffer);
7015
+ pm_buffer_append_string(output_buffer, "├── name:", 15);
7016
+ pm_buffer_append_byte(output_buffer, ' ');
7017
+ prettyprint_constant(output_buffer, parser, cast->name);
7018
+ pm_buffer_append_byte(output_buffer, '\n');
7019
+ }
7020
+
7021
+ // name_loc
7022
+ {
7023
+ pm_buffer_concat(output_buffer, prefix_buffer);
7024
+ pm_buffer_append_string(output_buffer, "└── name_loc:", 19);
7025
+ pm_location_t *location = &cast->name_loc;
7026
+ pm_buffer_append_byte(output_buffer, ' ');
7027
+ prettyprint_location(output_buffer, parser, location);
7028
+ pm_buffer_append_string(output_buffer, " = \"", 4);
7029
+ prettyprint_source(output_buffer, location->start, (size_t) (location->end - location->start));
7030
+ pm_buffer_append_string(output_buffer, "\"\n", 2);
7031
+ }
7032
+
7033
+ break;
7034
+ }
7010
7035
  case PM_REQUIRED_PARAMETER_NODE: {
7011
7036
  pm_required_parameter_node_t *cast = (pm_required_parameter_node_t *) node;
7012
7037
  pm_buffer_append_string(output_buffer, "@ RequiredParameterNode (location: ", 35);
@@ -8211,7 +8236,9 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
8211
8236
  }
8212
8237
  }
8213
8238
 
8214
- // Pretty-prints the AST represented by the given node to the given buffer.
8239
+ /**
8240
+ * Pretty-prints the AST represented by the given node to the given buffer.
8241
+ */
8215
8242
  PRISM_EXPORTED_FUNCTION void
8216
8243
  pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node) {
8217
8244
  pm_buffer_t prefix_buffer = { 0 };