prism 0.16.0 → 0.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -1
  3. data/Makefile +6 -0
  4. data/README.md +1 -1
  5. data/config.yml +50 -35
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/serialization.md +28 -29
  8. data/ext/prism/api_node.c +802 -770
  9. data/ext/prism/api_pack.c +20 -9
  10. data/ext/prism/extension.c +464 -162
  11. data/ext/prism/extension.h +1 -1
  12. data/include/prism/ast.h +3173 -763
  13. data/include/prism/defines.h +32 -9
  14. data/include/prism/diagnostic.h +36 -3
  15. data/include/prism/enc/pm_encoding.h +118 -28
  16. data/include/prism/node.h +38 -13
  17. data/include/prism/options.h +204 -0
  18. data/include/prism/pack.h +44 -33
  19. data/include/prism/parser.h +445 -200
  20. data/include/prism/prettyprint.h +12 -1
  21. data/include/prism/regexp.h +16 -2
  22. data/include/prism/util/pm_buffer.h +94 -16
  23. data/include/prism/util/pm_char.h +162 -48
  24. data/include/prism/util/pm_constant_pool.h +126 -32
  25. data/include/prism/util/pm_list.h +68 -38
  26. data/include/prism/util/pm_memchr.h +18 -3
  27. data/include/prism/util/pm_newline_list.h +70 -27
  28. data/include/prism/util/pm_state_stack.h +25 -7
  29. data/include/prism/util/pm_string.h +115 -27
  30. data/include/prism/util/pm_string_list.h +25 -6
  31. data/include/prism/util/pm_strncasecmp.h +32 -0
  32. data/include/prism/util/pm_strpbrk.h +31 -17
  33. data/include/prism/version.h +27 -2
  34. data/include/prism.h +224 -31
  35. data/lib/prism/compiler.rb +6 -3
  36. data/lib/prism/debug.rb +23 -7
  37. data/lib/prism/dispatcher.rb +33 -18
  38. data/lib/prism/dsl.rb +10 -5
  39. data/lib/prism/ffi.rb +132 -80
  40. data/lib/prism/lex_compat.rb +25 -15
  41. data/lib/prism/mutation_compiler.rb +10 -5
  42. data/lib/prism/node.rb +370 -135
  43. data/lib/prism/node_ext.rb +1 -1
  44. data/lib/prism/node_inspector.rb +1 -1
  45. data/lib/prism/pack.rb +79 -40
  46. data/lib/prism/parse_result/comments.rb +7 -2
  47. data/lib/prism/parse_result/newlines.rb +4 -0
  48. data/lib/prism/parse_result.rb +150 -30
  49. data/lib/prism/pattern.rb +11 -0
  50. data/lib/prism/ripper_compat.rb +28 -10
  51. data/lib/prism/serialize.rb +86 -54
  52. data/lib/prism/visitor.rb +10 -3
  53. data/lib/prism.rb +20 -2
  54. data/prism.gemspec +4 -2
  55. data/rbi/prism.rbi +104 -60
  56. data/rbi/prism_static.rbi +16 -2
  57. data/sig/prism.rbs +72 -43
  58. data/sig/prism_static.rbs +14 -1
  59. data/src/diagnostic.c +56 -53
  60. data/src/enc/pm_big5.c +1 -0
  61. data/src/enc/pm_euc_jp.c +1 -0
  62. data/src/enc/pm_gbk.c +1 -0
  63. data/src/enc/pm_shift_jis.c +1 -0
  64. data/src/enc/pm_tables.c +316 -80
  65. data/src/enc/pm_unicode.c +53 -8
  66. data/src/enc/pm_windows_31j.c +1 -0
  67. data/src/node.c +334 -321
  68. data/src/options.c +170 -0
  69. data/src/prettyprint.c +74 -47
  70. data/src/prism.c +1642 -856
  71. data/src/regexp.c +151 -95
  72. data/src/serialize.c +44 -20
  73. data/src/token_type.c +3 -1
  74. data/src/util/pm_buffer.c +45 -15
  75. data/src/util/pm_char.c +103 -57
  76. data/src/util/pm_constant_pool.c +51 -21
  77. data/src/util/pm_list.c +12 -4
  78. data/src/util/pm_memchr.c +5 -3
  79. data/src/util/pm_newline_list.c +20 -12
  80. data/src/util/pm_state_stack.c +9 -3
  81. data/src/util/pm_string.c +95 -85
  82. data/src/util/pm_string_list.c +14 -15
  83. data/src/util/pm_strncasecmp.c +10 -3
  84. data/src/util/pm_strpbrk.c +25 -19
  85. metadata +5 -3
  86. data/docs/prism.png +0 -0
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file pm_string.h
3
+ *
4
+ * A generic string type that can have various ownership semantics.
5
+ */
1
6
  #ifndef PRISM_STRING_H
2
7
  #define PRISM_STRING_H
3
8
 
@@ -9,54 +14,137 @@
9
14
  #include <stdlib.h>
10
15
  #include <string.h>
11
16
 
12
- // This struct represents a string value.
17
+ // The following headers are necessary to read files using demand paging.
18
+ #ifdef _WIN32
19
+ #include <windows.h>
20
+ #else
21
+ #include <fcntl.h>
22
+ #include <sys/mman.h>
23
+ #include <sys/stat.h>
24
+ #include <unistd.h>
25
+ #endif
26
+
27
+ /**
28
+ * A generic string type that can have various ownership semantics.
29
+ */
13
30
  typedef struct {
31
+ /** A pointer to the start of the string. */
14
32
  const uint8_t *source;
33
+
34
+ /** The length of the string in bytes of memory. */
15
35
  size_t length;
16
- // This field is not the first one, because otherwise things like .pm_string_t_field = 123/pm_constant_id_t does not warn or error
17
- enum { PM_STRING_SHARED, PM_STRING_OWNED, PM_STRING_CONSTANT, PM_STRING_MAPPED } type;
36
+
37
+ /** The type of the string. This field determines how the string should be freed. */
38
+ enum {
39
+ /** This string is a constant string, and should not be freed. */
40
+ PM_STRING_CONSTANT,
41
+
42
+ /** This is a slice of another string, and should not be freed. */
43
+ PM_STRING_SHARED,
44
+
45
+ /** This string owns its memory, and should be freed using `pm_string_free`. */
46
+ PM_STRING_OWNED,
47
+
48
+ /** This string is a memory-mapped file, and should be freed using `pm_string_free`. */
49
+ PM_STRING_MAPPED
50
+ } type;
18
51
  } pm_string_t;
19
52
 
20
- #define PM_EMPTY_STRING ((pm_string_t) { .type = PM_STRING_CONSTANT, .source = NULL, .length = 0 })
53
+ /**
54
+ * Returns the size of the pm_string_t struct. This is necessary to allocate the
55
+ * correct amount of memory in the FFI backend.
56
+ *
57
+ * @return The size of the pm_string_t struct.
58
+ */
59
+ PRISM_EXPORTED_FUNCTION size_t pm_string_sizeof(void);
60
+
61
+ /**
62
+ * Defines an empty string. This is useful for initializing a string that will
63
+ * be filled in later.
64
+ */
65
+ #define PM_STRING_EMPTY ((pm_string_t) { .type = PM_STRING_CONSTANT, .source = NULL, .length = 0 })
21
66
 
22
- // Initialize a shared string that is based on initial input.
67
+ /**
68
+ * Initialize a shared string that is based on initial input.
69
+ *
70
+ * @param string The string to initialize.
71
+ * @param start The start of the string.
72
+ * @param end The end of the string.
73
+ */
23
74
  void pm_string_shared_init(pm_string_t *string, const uint8_t *start, const uint8_t *end);
24
75
 
25
- // Initialize an owned string that is responsible for freeing allocated memory.
76
+ /**
77
+ * Initialize an owned string that is responsible for freeing allocated memory.
78
+ *
79
+ * @param string The string to initialize.
80
+ * @param source The source of the string.
81
+ * @param length The length of the string.
82
+ */
26
83
  void pm_string_owned_init(pm_string_t *string, uint8_t *source, size_t length);
27
84
 
28
- // Initialize a constant string that doesn't own its memory source.
85
+ /**
86
+ * Initialize a constant string that doesn't own its memory source.
87
+ *
88
+ * @param string The string to initialize.
89
+ * @param source The source of the string.
90
+ * @param length The length of the string.
91
+ */
29
92
  void pm_string_constant_init(pm_string_t *string, const char *source, size_t length);
30
93
 
31
- // Read the file indicated by the filepath parameter into source and load its
32
- // contents and size into the given pm_string_t.
33
- // The given pm_string_t should be freed using pm_string_free() when it is no longer used.
34
- //
35
- // We want to use demand paging as much as possible in order to avoid having to
36
- // read the entire file into memory (which could be detrimental to performance
37
- // for large files). This means that if we're on windows we'll use
38
- // `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
39
- // `mmap`, and on other POSIX systems we'll use `read`.
94
+ /**
95
+ * Read the file indicated by the filepath parameter into source and load its
96
+ * contents and size into the given `pm_string_t`. The given `pm_string_t`
97
+ * should be freed using `pm_string_free` when it is no longer used.
98
+ *
99
+ * We want to use demand paging as much as possible in order to avoid having to
100
+ * read the entire file into memory (which could be detrimental to performance
101
+ * for large files). This means that if we're on windows we'll use
102
+ * `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
103
+ * `mmap`, and on other POSIX systems we'll use `read`.
104
+ *
105
+ * @param string The string to initialize.
106
+ * @param filepath The filepath to read.
107
+ * @return Whether or not the file was successfully mapped.
108
+ */
40
109
  PRISM_EXPORTED_FUNCTION bool pm_string_mapped_init(pm_string_t *string, const char *filepath);
41
110
 
42
- // Returns the memory size associated with the string.
111
+ /**
112
+ * Returns the memory size associated with the string.
113
+ *
114
+ * @param string The string to get the memory size of.
115
+ * @return The size of the memory associated with the string.
116
+ */
43
117
  size_t pm_string_memsize(const pm_string_t *string);
44
118
 
45
- // Ensure the string is owned. If it is not, then reinitialize it as owned and
46
- // copy over the previous source.
119
+ /**
120
+ * Ensure the string is owned. If it is not, then reinitialize it as owned and
121
+ * copy over the previous source.
122
+ *
123
+ * @param string The string to ensure is owned.
124
+ */
47
125
  void pm_string_ensure_owned(pm_string_t *string);
48
126
 
49
- // Returns the length associated with the string.
127
+ /**
128
+ * Returns the length associated with the string.
129
+ *
130
+ * @param string The string to get the length of.
131
+ * @return The length of the string.
132
+ */
50
133
  PRISM_EXPORTED_FUNCTION size_t pm_string_length(const pm_string_t *string);
51
134
 
52
- // Returns the start pointer associated with the string.
135
+ /**
136
+ * Returns the start pointer associated with the string.
137
+ *
138
+ * @param string The string to get the start pointer of.
139
+ * @return The start pointer of the string.
140
+ */
53
141
  PRISM_EXPORTED_FUNCTION const uint8_t * pm_string_source(const pm_string_t *string);
54
142
 
55
- // Free the associated memory of the given string.
143
+ /**
144
+ * Free the associated memory of the given string.
145
+ *
146
+ * @param string The string to free.
147
+ */
56
148
  PRISM_EXPORTED_FUNCTION void pm_string_free(pm_string_t *string);
57
149
 
58
- // Returns the size of the pm_string_t struct. This is necessary to allocate the
59
- // correct amount of memory in the FFI backend.
60
- PRISM_EXPORTED_FUNCTION size_t pm_string_sizeof(void);
61
-
62
- #endif // PRISM_STRING_H
150
+ #endif
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file pm_string_list.h
3
+ *
4
+ * A list of strings.
5
+ */
1
6
  #ifndef PRISM_STRING_LIST_H
2
7
  #define PRISM_STRING_LIST_H
3
8
 
@@ -7,19 +12,33 @@
7
12
  #include <stddef.h>
8
13
  #include <stdlib.h>
9
14
 
15
+ /**
16
+ * A list of strings.
17
+ */
10
18
  typedef struct {
11
- pm_string_t *strings;
19
+ /** The length of the string list. */
12
20
  size_t length;
21
+
22
+ /** The capacity of the string list that has been allocated. */
13
23
  size_t capacity;
14
- } pm_string_list_t;
15
24
 
16
- // Initialize a pm_string_list_t with its default values.
17
- PRISM_EXPORTED_FUNCTION void pm_string_list_init(pm_string_list_t *string_list);
25
+ /** A pointer to the start of the string list. */
26
+ pm_string_t *strings;
27
+ } pm_string_list_t;
18
28
 
19
- // Append a pm_string_t to the given string list.
29
+ /**
30
+ * Append a pm_string_t to the given string list.
31
+ *
32
+ * @param string_list The string list to append to.
33
+ * @param string The string to append.
34
+ */
20
35
  void pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string);
21
36
 
22
- // Free the memory associated with the string list.
37
+ /**
38
+ * Free the memory associated with the string list.
39
+ *
40
+ * @param string_list The string list to free.
41
+ */
23
42
  PRISM_EXPORTED_FUNCTION void pm_string_list_free(pm_string_list_t *string_list);
24
43
 
25
44
  #endif
@@ -0,0 +1,32 @@
1
+ /**
2
+ * @file pm_strncasecmp.h
3
+ *
4
+ * A custom strncasecmp implementation.
5
+ */
6
+ #ifndef PRISM_STRNCASECMP_H
7
+ #define PRISM_STRNCASECMP_H
8
+
9
+ #include "prism/defines.h"
10
+
11
+ #include <ctype.h>
12
+ #include <stddef.h>
13
+ #include <stdint.h>
14
+
15
+ /**
16
+ * Compare two strings, ignoring case, up to the given length. Returns 0 if the
17
+ * strings are equal, a negative number if string1 is less than string2, or a
18
+ * positive number if string1 is greater than string2.
19
+ *
20
+ * Note that this is effectively our own implementation of strncasecmp, but it's
21
+ * not available on all of the platforms we want to support so we're rolling it
22
+ * here.
23
+ *
24
+ * @param string1 The first string to compare.
25
+ * @param string2 The second string to compare
26
+ * @param length The maximum number of characters to compare.
27
+ * @return 0 if the strings are equal, a negative number if string1 is less than
28
+ * string2, or a positive number if string1 is greater than string2.
29
+ */
30
+ int pm_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length);
31
+
32
+ #endif
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file pm_strpbrk.h
3
+ *
4
+ * A custom strpbrk implementation.
5
+ */
1
6
  #ifndef PRISM_STRPBRK_H
2
7
  #define PRISM_STRPBRK_H
3
8
 
@@ -7,23 +12,32 @@
7
12
  #include <stddef.h>
8
13
  #include <string.h>
9
14
 
10
- // Here we have rolled our own version of strpbrk. The standard library strpbrk
11
- // has undefined behavior when the source string is not null-terminated. We want
12
- // to support strings that are not null-terminated because pm_parse does not
13
- // have the contract that the string is null-terminated. (This is desirable
14
- // because it means the extension can call pm_parse with the result of a call to
15
- // mmap).
16
- //
17
- // The standard library strpbrk also does not support passing a maximum length
18
- // to search. We want to support this for the reason mentioned above, but we
19
- // also don't want it to stop on null bytes. Ruby actually allows null bytes
20
- // within strings, comments, regular expressions, etc. So we need to be able to
21
- // skip past them.
22
- //
23
- // Finally, we want to support encodings wherein the charset could contain
24
- // characters that are trailing bytes of multi-byte characters. For example, in
25
- // Shift-JIS, the backslash character can be a trailing byte. In that case we
26
- // need to take a slower path and iterate one multi-byte character at a time.
15
+ /**
16
+ * Here we have rolled our own version of strpbrk. The standard library strpbrk
17
+ * has undefined behavior when the source string is not null-terminated. We want
18
+ * to support strings that are not null-terminated because pm_parse does not
19
+ * have the contract that the string is null-terminated. (This is desirable
20
+ * because it means the extension can call pm_parse with the result of a call to
21
+ * mmap).
22
+ *
23
+ * The standard library strpbrk also does not support passing a maximum length
24
+ * to search. We want to support this for the reason mentioned above, but we
25
+ * also don't want it to stop on null bytes. Ruby actually allows null bytes
26
+ * within strings, comments, regular expressions, etc. So we need to be able to
27
+ * skip past them.
28
+ *
29
+ * Finally, we want to support encodings wherein the charset could contain
30
+ * characters that are trailing bytes of multi-byte characters. For example, in
31
+ * Shift-JIS, the backslash character can be a trailing byte. In that case we
32
+ * need to take a slower path and iterate one multi-byte character at a time.
33
+ *
34
+ * @param parser The parser.
35
+ * @param source The source string.
36
+ * @param charset The charset to search for.
37
+ * @param length The maximum length to search.
38
+ * @return A pointer to the first character in the source string that is in the
39
+ * charset, or NULL if no such character exists.
40
+ */
27
41
  const uint8_t * pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length);
28
42
 
29
43
  #endif
@@ -1,4 +1,29 @@
1
+ /**
2
+ * @file version.h
3
+ *
4
+ * The version of the Prism library.
5
+ */
6
+ #ifndef PRISM_VERSION_H
7
+ #define PRISM_VERSION_H
8
+
9
+ /**
10
+ * The major version of the Prism library as an int.
11
+ */
1
12
  #define PRISM_VERSION_MAJOR 0
2
- #define PRISM_VERSION_MINOR 16
13
+
14
+ /**
15
+ * The minor version of the Prism library as an int.
16
+ */
17
+ #define PRISM_VERSION_MINOR 17
18
+
19
+ /**
20
+ * The patch version of the Prism library as an int.
21
+ */
3
22
  #define PRISM_VERSION_PATCH 0
4
- #define PRISM_VERSION "0.16.0"
23
+
24
+ /**
25
+ * The version of the Prism library as a constant string.
26
+ */
27
+ #define PRISM_VERSION "0.17.0"
28
+
29
+ #endif
data/include/prism.h CHANGED
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file prism.h
3
+ *
4
+ * The main header file for the prism parser.
5
+ */
1
6
  #ifndef PRISM_H
2
7
  #define PRISM_H
3
8
 
@@ -5,10 +10,12 @@
5
10
  #include "prism/util/pm_buffer.h"
6
11
  #include "prism/util/pm_char.h"
7
12
  #include "prism/util/pm_memchr.h"
13
+ #include "prism/util/pm_strncasecmp.h"
8
14
  #include "prism/util/pm_strpbrk.h"
9
15
  #include "prism/ast.h"
10
16
  #include "prism/diagnostic.h"
11
17
  #include "prism/node.h"
18
+ #include "prism/options.h"
12
19
  #include "prism/pack.h"
13
20
  #include "prism/parser.h"
14
21
  #include "prism/prettyprint.h"
@@ -28,54 +35,240 @@
28
35
  #include <strings.h>
29
36
  #endif
30
37
 
31
- void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
32
-
33
- void pm_serialize_encoding(pm_encoding_t *encoding, pm_buffer_t *buffer);
34
-
35
- void pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer);
36
-
37
- void pm_parser_metadata(pm_parser_t *parser, const char *metadata);
38
-
39
- // The prism version and the serialization format.
38
+ /**
39
+ * The prism version and the serialization format.
40
+ *
41
+ * @returns The prism version as a constant string.
42
+ */
40
43
  PRISM_EXPORTED_FUNCTION const char * pm_version(void);
41
44
 
42
- // Initialize a parser with the given start and end pointers.
43
- PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const char *filepath);
45
+ /**
46
+ * Initialize a parser with the given start and end pointers.
47
+ *
48
+ * @param parser The parser to initialize.
49
+ * @param source The source to parse.
50
+ * @param size The size of the source.
51
+ * @param options The optional options to use when parsing.
52
+ */
53
+ PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm_options_t *options);
44
54
 
45
- // Register a callback that will be called whenever prism changes the encoding it
46
- // is using to parse based on the magic comment.
55
+ /**
56
+ * Register a callback that will be called whenever prism changes the encoding
57
+ * it is using to parse based on the magic comment.
58
+ *
59
+ * @param parser The parser to register the callback with.
60
+ * @param callback The callback to register.
61
+ */
47
62
  PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_changed_callback(pm_parser_t *parser, pm_encoding_changed_callback_t callback);
48
63
 
49
- // Register a callback that will be called when prism encounters a magic comment
50
- // with an encoding referenced that it doesn't understand. The callback should
51
- // return NULL if it also doesn't understand the encoding or it should return a
52
- // pointer to a pm_encoding_t struct that contains the functions necessary to
53
- // parse identifiers.
64
+ /**
65
+ * Register a callback that will be called when prism encounters a magic comment
66
+ * with an encoding referenced that it doesn't understand. The callback should
67
+ * return NULL if it also doesn't understand the encoding or it should return a
68
+ * pointer to a pm_encoding_t struct that contains the functions necessary to
69
+ * parse identifiers.
70
+ *
71
+ * @param parser The parser to register the callback with.
72
+ * @param callback The callback to register.
73
+ */
54
74
  PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_decode_callback(pm_parser_t *parser, pm_encoding_decode_callback_t callback);
55
75
 
56
- // Free any memory associated with the given parser.
76
+ /**
77
+ * Free any memory associated with the given parser.
78
+ *
79
+ * @param parser The parser to free.
80
+ */
57
81
  PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser);
58
82
 
59
- // Parse the Ruby source associated with the given parser and return the tree.
83
+ /**
84
+ * Initiate the parser with the given parser.
85
+ *
86
+ * @param parser The parser to use.
87
+ * @return The AST representing the source.
88
+ */
60
89
  PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse(pm_parser_t *parser);
61
90
 
62
- // Serialize the AST represented by the given node to the given buffer.
91
+ /**
92
+ * Serialize the given list of comments to the given buffer.
93
+ *
94
+ * @param parser The parser to serialize.
95
+ * @param list The list of comments to serialize.
96
+ * @param buffer The buffer to serialize to.
97
+ */
98
+ void pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer);
99
+
100
+ /**
101
+ * Serialize the name of the encoding to the buffer.
102
+ *
103
+ * @param encoding The encoding to serialize.
104
+ * @param buffer The buffer to serialize to.
105
+ */
106
+ void pm_serialize_encoding(pm_encoding_t *encoding, pm_buffer_t *buffer);
107
+
108
+ /**
109
+ * Serialize the encoding, metadata, nodes, and constant pool.
110
+ *
111
+ * @param parser The parser to serialize.
112
+ * @param node The node to serialize.
113
+ * @param buffer The buffer to serialize to.
114
+ */
115
+ void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
116
+
117
+ /**
118
+ * Serialize the AST represented by the given node to the given buffer.
119
+ *
120
+ * @param parser The parser to serialize.
121
+ * @param node The node to serialize.
122
+ * @param buffer The buffer to serialize to.
123
+ */
63
124
  PRISM_EXPORTED_FUNCTION void pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
64
125
 
65
- // Parse the given source to the AST and serialize the AST to the given buffer.
66
- PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
126
+ /**
127
+ * Parse the given source to the AST and dump the AST to the given buffer.
128
+ *
129
+ * @param buffer The buffer to serialize to.
130
+ * @param source The source to parse.
131
+ * @param size The size of the source.
132
+ * @param data The optional data to pass to the parser.
133
+ */
134
+ PRISM_EXPORTED_FUNCTION void pm_serialize_parse(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
67
135
 
68
- // Parse and serialize the comments in the given source to the given buffer.
69
- PRISM_EXPORTED_FUNCTION void pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
136
+ /**
137
+ * Parse and serialize the comments in the given source to the given buffer.
138
+ *
139
+ * @param buffer The buffer to serialize to.
140
+ * @param source The source to parse.
141
+ * @param size The size of the source.
142
+ * @param data The optional data to pass to the parser.
143
+ */
144
+ PRISM_EXPORTED_FUNCTION void pm_serialize_parse_comments(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
70
145
 
71
- // Lex the given source and serialize to the given buffer.
72
- PRISM_EXPORTED_FUNCTION void pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_buffer_t *buffer);
146
+ /**
147
+ * Lex the given source and serialize to the given buffer.
148
+ *
149
+ * @param source The source to lex.
150
+ * @param size The size of the source.
151
+ * @param buffer The buffer to serialize to.
152
+ * @param data The optional data to pass to the lexer.
153
+ */
154
+ PRISM_EXPORTED_FUNCTION void pm_serialize_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
73
155
 
74
- // Parse and serialize both the AST and the tokens represented by the given
75
- // source to the given buffer.
76
- PRISM_EXPORTED_FUNCTION void pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
156
+ /**
157
+ * Parse and serialize both the AST and the tokens represented by the given
158
+ * source to the given buffer.
159
+ *
160
+ * @param buffer The buffer to serialize to.
161
+ * @param source The source to parse.
162
+ * @param size The size of the source.
163
+ * @param data The optional data to pass to the parser.
164
+ */
165
+ PRISM_EXPORTED_FUNCTION void pm_serialize_parse_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
77
166
 
78
- // Returns a string representation of the given token type.
167
+ /**
168
+ * Returns a string representation of the given token type.
169
+ *
170
+ * @param token_type The token type to convert to a string.
171
+ * @return A string representation of the given token type.
172
+ */
79
173
  PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_type);
80
174
 
175
+ /**
176
+ * @mainpage
177
+ *
178
+ * Prism is a parser for the Ruby programming language. It is designed to be
179
+ * portable, error tolerant, and maintainable. It is written in C99 and has no
180
+ * dependencies. It is currently being integrated into
181
+ * [CRuby](https://github.com/ruby/ruby),
182
+ * [JRuby](https://github.com/jruby/jruby),
183
+ * [TruffleRuby](https://github.com/oracle/truffleruby),
184
+ * [Sorbet](https://github.com/sorbet/sorbet), and
185
+ * [Syntax Tree](https://github.com/ruby-syntax-tree/syntax_tree).
186
+ *
187
+ * @section getting-started Getting started
188
+ *
189
+ * If you're vendoring this project and compiling it statically then as long as
190
+ * you have a C99 compiler you will be fine. If you're linking against it as
191
+ * shared library, then you should compile with `-fvisibility=hidden` and
192
+ * `-DPRISM_EXPORT_SYMBOLS` to tell prism to make only its public interface
193
+ * visible.
194
+ *
195
+ * @section parsing Parsing
196
+ *
197
+ * In order to parse Ruby code, the structures and functions that you're going
198
+ * to want to use and be aware of are:
199
+ *
200
+ * * `pm_parser_t` - the main parser structure
201
+ * * `pm_parser_init` - initialize a parser
202
+ * * `pm_parse` - parse and return the root node
203
+ * * `pm_node_destroy` - deallocate the root node returned by `pm_parse`
204
+ * * `pm_parser_free` - free the internal memory of the parser
205
+ *
206
+ * Putting all of this together would look something like:
207
+ *
208
+ * ```c
209
+ * void parse(const uint8_t *source, size_t length) {
210
+ * pm_parser_t parser;
211
+ * pm_parser_init(&parser, source, length, NULL);
212
+ *
213
+ * pm_node_t *root = pm_parse(&parser);
214
+ * printf("PARSED!\n");
215
+ *
216
+ * pm_node_destroy(root);
217
+ * pm_parser_free(&parser);
218
+ * }
219
+ * ```
220
+ *
221
+ * All of the nodes "inherit" from `pm_node_t` by embedding those structures as
222
+ * their first member. This means you can downcast and upcast any node in the
223
+ * tree to a `pm_node_t`.
224
+ *
225
+ * @section serializing Serializing
226
+ *
227
+ * Prism provides the ability to serialize the AST and its related metadata into
228
+ * a binary format. This format is designed to be portable to different
229
+ * languages and runtimes so that you only need to make one FFI call in order to
230
+ * parse Ruby code. The structures and functions that you're going to want to
231
+ * use and be aware of are:
232
+ *
233
+ * * `pm_buffer_t` - a small buffer object that will hold the serialized AST
234
+ * * `pm_buffer_free` - free the memory associated with the buffer
235
+ * * `pm_serialize` - serialize the AST into a buffer
236
+ * * `pm_serialize_parse` - parse and serialize the AST into a buffer
237
+ *
238
+ * Putting all of this together would look something like:
239
+ *
240
+ * ```c
241
+ * void serialize(const uint8_t *source, size_t length) {
242
+ * pm_buffer_t buffer = { 0 };
243
+ *
244
+ * pm_serialize_parse(&buffer, source, length, NULL);
245
+ * printf("SERIALIZED!\n");
246
+ *
247
+ * pm_buffer_free(&buffer);
248
+ * }
249
+ * ```
250
+ *
251
+ * @section inspecting Inspecting
252
+ *
253
+ * Prism provides the ability to inspect the AST by pretty-printing nodes. You
254
+ * can do this with the `pm_prettyprint` function, which you would use like:
255
+ *
256
+ * ```c
257
+ * void prettyprint(const uint8_t *source, size_t length) {
258
+ * pm_parser_t parser;
259
+ * pm_parser_init(&parser, source, length, NULL);
260
+ *
261
+ * pm_node_t *root = pm_parse(&parser);
262
+ * pm_buffer_t buffer = { 0 };
263
+ *
264
+ * pm_prettyprint(&buffer, &parser, root);
265
+ * printf("*.s%\n", (int) buffer.length, buffer.value);
266
+ *
267
+ * pm_buffer_free(&buffer);
268
+ * pm_node_destroy(root);
269
+ * pm_parser_free(&parser);
270
+ * }
271
+ * ```
272
+ */
273
+
81
274
  #endif
@@ -291,9 +291,6 @@ module Prism
291
291
  # Compile a KeywordHashNode node
292
292
  alias visit_keyword_hash_node visit_child_nodes
293
293
 
294
- # Compile a KeywordParameterNode node
295
- alias visit_keyword_parameter_node visit_child_nodes
296
-
297
294
  # Compile a KeywordRestParameterNode node
298
295
  alias visit_keyword_rest_parameter_node visit_child_nodes
299
296
 
@@ -354,6 +351,9 @@ module Prism
354
351
  # Compile a NumberedReferenceReadNode node
355
352
  alias visit_numbered_reference_read_node visit_child_nodes
356
353
 
354
+ # Compile a OptionalKeywordParameterNode node
355
+ alias visit_optional_keyword_parameter_node visit_child_nodes
356
+
357
357
  # Compile a OptionalParameterNode node
358
358
  alias visit_optional_parameter_node visit_child_nodes
359
359
 
@@ -393,6 +393,9 @@ module Prism
393
393
  # Compile a RegularExpressionNode node
394
394
  alias visit_regular_expression_node visit_child_nodes
395
395
 
396
+ # Compile a RequiredKeywordParameterNode node
397
+ alias visit_required_keyword_parameter_node visit_child_nodes
398
+
396
399
  # Compile a RequiredParameterNode node
397
400
  alias visit_required_parameter_node visit_child_nodes
398
401