prism 2.0.0.rc1-arm-linux-gnu → 2.1.0-arm-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +61 -41
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +8 -8
  6. data/README.md +3 -1
  7. data/docs/fuzzing.md +5 -5
  8. data/docs/serialization.md +20 -6
  9. data/ext/prism/api_node.c +81 -81
  10. data/ext/prism/extension.c +54 -69
  11. data/ext/prism/extension.h +1 -1
  12. data/include/prism/arena.h +8 -0
  13. data/include/prism/arena.hpp +42 -0
  14. data/include/prism/ast.h +11 -1
  15. data/include/prism/ast.hpp +641 -0
  16. data/include/prism/buffer.h +8 -0
  17. data/include/prism/buffer.hpp +55 -0
  18. data/include/prism/comments.h +17 -0
  19. data/include/prism/compiler/accel.h +3 -1
  20. data/include/prism/compiler/branch.h +26 -0
  21. data/include/prism/compiler/bswap.h +46 -0
  22. data/include/prism/compiler/cplusplus.hpp +16 -0
  23. data/include/prism/compiler/endianness.h +34 -0
  24. data/include/prism/compiler/exceptions.hpp +17 -0
  25. data/include/prism/compiler/format.h +2 -0
  26. data/include/prism/compiler/lifetimebound.hpp +25 -0
  27. data/include/prism/constant_pool.h +25 -0
  28. data/include/prism/diagnostic.h +18 -0
  29. data/include/prism/errors_format.h +8 -0
  30. data/include/prism/internal/buffer.h +3 -0
  31. data/include/prism/internal/comments.h +2 -1
  32. data/include/prism/internal/node.h +8 -4
  33. data/include/prism/internal/parser.h +44 -13
  34. data/include/prism/internal/static_literals.h +24 -64
  35. data/include/prism/internal/stringy.h +1 -6
  36. data/include/prism/internal/xxh3.h +16 -0
  37. data/include/prism/json.h +8 -0
  38. data/include/prism/line_offset_list.h +8 -0
  39. data/include/prism/magic_comments.h +18 -0
  40. data/include/prism/node.h +55 -10
  41. data/include/prism/node.hpp +272 -0
  42. data/include/prism/node_type.hpp +38 -0
  43. data/include/prism/options.h +10 -1
  44. data/include/prism/options.hpp +43 -0
  45. data/include/prism/parser.h +93 -64
  46. data/include/prism/parser.hpp +274 -0
  47. data/include/prism/prettyprint.h +8 -0
  48. data/include/prism/serialize.h +8 -0
  49. data/include/prism/source.h +8 -0
  50. data/include/prism/stream.h +8 -0
  51. data/include/prism/string_query.h +19 -0
  52. data/include/prism/stringy.h +16 -27
  53. data/include/prism/stringy.hpp +31 -0
  54. data/include/prism/version.h +10 -2
  55. data/include/prism.h +0 -8
  56. data/include/prism.hpp +24 -0
  57. data/lib/prism/3.3/prism.so +0 -0
  58. data/lib/prism/3.4/prism.so +0 -0
  59. data/lib/prism/4.0/prism.so +0 -0
  60. data/lib/prism/ffi.rb +6 -0
  61. data/lib/prism/node.rb +736 -0
  62. data/lib/prism/parse_result.rb +23 -8
  63. data/lib/prism/serialize.rb +2568 -3287
  64. data/lib/prism/string_query.rb +8 -0
  65. data/prism.gemspec +21 -4
  66. data/rbi/generated/prism/node.rbi +2430 -968
  67. data/rbi/generated/prism/parse_result.rbi +16 -4
  68. data/rbi/generated/prism/serialize.rbi +8 -4
  69. data/rbi/generated/prism/string_query.rbi +7 -0
  70. data/sig/generated/prism/node.rbs +2439 -967
  71. data/sig/generated/prism/parse_result.rbs +16 -4
  72. data/sig/generated/prism/serialize.rbs +9 -4
  73. data/sig/generated/prism/string_query.rbs +7 -0
  74. data/src/arena.c +6 -1
  75. data/src/buffer.c +9 -0
  76. data/src/comments.c +11 -10
  77. data/src/diagnostic.c +24 -4
  78. data/src/integer.c +6 -1
  79. data/src/node.c +216 -368
  80. data/src/options.c +7 -13
  81. data/src/parser.c +70 -48
  82. data/src/prism.c +625 -462
  83. data/src/regexp.c +0 -1
  84. data/src/serialize.c +2301 -1950
  85. data/src/source.c +8 -2
  86. data/src/static_literals.c +280 -405
  87. data/src/string_query.c +14 -0
  88. data/src/stringy.c +3 -33
  89. data/src/xxh3.c +292 -0
  90. metadata +20 -3
@@ -12,6 +12,10 @@
12
12
 
13
13
  #include <stddef.h>
14
14
 
15
+ #ifdef __cplusplus
16
+ extern "C" {
17
+ #endif
18
+
15
19
  /**
16
20
  * A wrapper around a contiguous block of allocated memory.
17
21
  */
@@ -49,4 +53,8 @@ PRISM_EXPORTED_FUNCTION char * pm_buffer_value(const pm_buffer_t *buffer) PRISM_
49
53
  */
50
54
  PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(const pm_buffer_t *buffer) PRISM_NONNULL(1);
51
55
 
56
+ #ifdef __cplusplus
57
+ }
58
+ #endif
59
+
52
60
  #endif
@@ -0,0 +1,55 @@
1
+ /**
2
+ * @file buffer.hpp
3
+ *
4
+ * Ownership of buffers from C++, and access to their contents.
5
+ */
6
+ #ifndef PRISM_BUFFER_HPP
7
+ #define PRISM_BUFFER_HPP
8
+
9
+ #include "prism/compiler/cplusplus.hpp"
10
+ #include "prism/compiler/lifetimebound.hpp"
11
+
12
+ #include "prism/buffer.h"
13
+
14
+ #include <memory>
15
+ #include <string_view>
16
+
17
+ namespace prism {
18
+
19
+ /** Frees a buffer with pm_buffer_free. */
20
+ struct buffer_deleter {
21
+ /**
22
+ * Frees the given buffer.
23
+ *
24
+ * @param buffer The buffer to free.
25
+ */
26
+ void operator()(pm_buffer_t *buffer) const noexcept { pm_buffer_free(buffer); }
27
+ };
28
+
29
+ /** An owned buffer. */
30
+ using unique_buffer = std::unique_ptr<pm_buffer_t, buffer_deleter>;
31
+
32
+ /**
33
+ * Allocate a new empty buffer. Allocation failure aborts, so this never throws.
34
+ *
35
+ * @return A unique_buffer managing the newly allocated buffer.
36
+ */
37
+ [[nodiscard]] inline unique_buffer
38
+ make_buffer() noexcept {
39
+ return unique_buffer(pm_buffer_new());
40
+ }
41
+
42
+ /**
43
+ * The bytes written to the given buffer so far.
44
+ *
45
+ * @param buffer The buffer to view.
46
+ * @return A string_view of the contents of the buffer.
47
+ */
48
+ [[nodiscard]] inline std::string_view
49
+ view(const pm_buffer_t *buffer PRISM_LIFETIMEBOUND) noexcept {
50
+ return std::string_view(pm_buffer_value(buffer), pm_buffer_length(buffer));
51
+ }
52
+
53
+ }
54
+
55
+ #endif
@@ -14,6 +14,10 @@
14
14
 
15
15
  #include <stddef.h>
16
16
 
17
+ #ifdef __cplusplus
18
+ extern "C" {
19
+ #endif
20
+
17
21
  /** This is the type of a comment that we've found while parsing. */
18
22
  typedef enum {
19
23
  PM_COMMENT_INLINE,
@@ -52,4 +56,17 @@ PRISM_EXPORTED_FUNCTION pm_location_t pm_comment_location(const pm_comment_t *co
52
56
  */
53
57
  PRISM_EXPORTED_FUNCTION pm_comment_type_t pm_comment_type(const pm_comment_t *comment) PRISM_NONNULL(1);
54
58
 
59
+ /**
60
+ * Returns the comment that follows the given comment among the comments
61
+ * associated with the parser that found it.
62
+ *
63
+ * @param comment the comment whose successor we want to get
64
+ * @returns the next comment, or NULL if the given comment is the last one
65
+ */
66
+ PRISM_EXPORTED_FUNCTION const pm_comment_t * pm_comment_next(const pm_comment_t *comment) PRISM_NONNULL(1);
67
+
68
+ #ifdef __cplusplus
69
+ }
70
+ #endif
71
+
55
72
  #endif
@@ -4,6 +4,8 @@
4
4
  #ifndef PRISM_COMPILER_ACCEL_H
5
5
  #define PRISM_COMPILER_ACCEL_H
6
6
 
7
+ #include "prism/compiler/endianness.h"
8
+
7
9
  /**
8
10
  * Platform detection for SIMD/fast-path implementations. At most one of these
9
11
  * macros is defined, selecting the best available vectorization strategy.
@@ -17,7 +19,7 @@
17
19
  # define PRISM_HAS_NEON
18
20
  #elif (defined(__x86_64__) && defined(__SSE2__)) || (defined(_MSC_VER) && defined(_M_X64))
19
21
  # define PRISM_HAS_SSE2
20
- #elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
22
+ #elif defined(PRISM_LITTLE_ENDIAN_COMPTIME)
21
23
  # define PRISM_HAS_SWAR
22
24
  #endif
23
25
 
@@ -0,0 +1,26 @@
1
+ /**
2
+ * @file compiler/branch.h
3
+ */
4
+ #ifndef PRISM_COMPILER_BRANCH_H
5
+ #define PRISM_COMPILER_BRANCH_H
6
+
7
+ /**
8
+ * Branch prediction hints. These tell the compiler which side of a branch to
9
+ * predicate the fast path through, so it can lay out instructions to favour
10
+ * that side and steer the static branch predictor.
11
+ */
12
+ #if defined(__GNUC__) || defined(__clang__)
13
+ /** The compiler should predicate that this branch will be taken. */
14
+ #define PRISM_LIKELY(x) __builtin_expect(!!(x), 1)
15
+
16
+ /** The compiler should predicate that this branch will not be taken. */
17
+ #define PRISM_UNLIKELY(x) __builtin_expect(!!(x), 0)
18
+ #else
19
+ /** Void because this platform does not support branch prediction hints. */
20
+ #define PRISM_LIKELY(x) (x)
21
+
22
+ /** Void because this platform does not support branch prediction hints. */
23
+ #define PRISM_UNLIKELY(x) (x)
24
+ #endif
25
+
26
+ #endif
@@ -0,0 +1,46 @@
1
+ /**
2
+ * @file compiler/bswap.h
3
+ */
4
+ #ifndef PRISM_COMPILER_BSWAP_H
5
+ #define PRISM_COMPILER_BSWAP_H
6
+
7
+ /**
8
+ * Byte swap utilities for 32-bit and 64-bit integers.
9
+ */
10
+ #if defined(__GNUC__) || defined(__clang__)
11
+ #define pm_bswap32(x_) __builtin_bswap32((x_))
12
+ #define pm_bswap64(x_) __builtin_bswap64((x_))
13
+ #elif defined(_MSC_VER)
14
+ #include <stdlib.h>
15
+
16
+ #define pm_bswap32(x_) _byteswap_ulong((x_))
17
+ #define pm_bswap64(x_) _byteswap_uint64((x_))
18
+ #else
19
+ #include <stdint.h>
20
+
21
+ static uint32_t
22
+ pm_bswap32(uint32_t x) {
23
+ return (
24
+ ((x >> 24) & 0x000000FFUL) |
25
+ ((x << 24) & 0xFF000000UL) |
26
+ ((x >> 8) & 0x0000FF00UL) |
27
+ ((x << 8) & 0x00FF0000UL)
28
+ );
29
+ }
30
+
31
+ static uint64_t
32
+ pm_bswap64(uint64_t x) {
33
+ return (
34
+ ((x >> 56) & 0x00000000000000FFULL) |
35
+ ((x << 56) & 0xFF00000000000000ULL) |
36
+ ((x >> 40) & 0x000000000000FF00ULL) |
37
+ ((x << 40) & 0x00FF000000000000ULL) |
38
+ ((x >> 24) & 0x0000000000FF0000ULL) |
39
+ ((x << 24) & 0x0000FF0000000000ULL) |
40
+ ((x >> 8) & 0x00000000FF000000ULL) |
41
+ ((x << 8) & 0x000000FF00000000ULL)
42
+ );
43
+ }
44
+ #endif
45
+
46
+ #endif
@@ -0,0 +1,16 @@
1
+ /**
2
+ * @file compiler/cplusplus.hpp
3
+ */
4
+ #ifndef PRISM_COMPILER_CPLUSPLUS_HPP
5
+ #define PRISM_COMPILER_CPLUSPLUS_HPP
6
+
7
+ /**
8
+ * The C++ interface relies on C++17 for inline variables, std::string_view,
9
+ * std::void_t, and std::is_invocable_r_v. MSVC reports __cplusplus as 199711L
10
+ * unless /Zc:__cplusplus is passed, so its own _MSVC_LANG is checked as well.
11
+ */
12
+ #if !defined(__cplusplus) || ((__cplusplus < 201703L) && (!defined(_MSVC_LANG) || (_MSVC_LANG < 201703L)))
13
+ # error "the prism C++ interface requires C++17 or later"
14
+ #endif
15
+
16
+ #endif
@@ -0,0 +1,34 @@
1
+ /**
2
+ * @file compiler/endianness.h
3
+ */
4
+ #ifndef PRISM_COMPILER_ENDIANNESS_H
5
+ #define PRISM_COMPILER_ENDIANNESS_H
6
+
7
+ /**
8
+ * Detect endianness at compile-time through the use of predefined macros.
9
+ */
10
+ #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
11
+ #define PRISM_LITTLE_ENDIAN_COMPTIME
12
+ #endif
13
+
14
+ /**
15
+ * Determine endianness at runtime, either through predefined macros or through
16
+ * a runtime check that is usually folded.
17
+ */
18
+ #if defined(PRISM_LITTLE_ENDIAN_COMPTIME)
19
+ #define PRISM_LITTLE_ENDIAN_RUNTIME 1
20
+ #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
21
+ #define PRISM_LITTLE_ENDIAN_RUNTIME 0
22
+ #else
23
+ #include <stdint.h>
24
+
25
+ static inline int
26
+ pm_little_endian(void) {
27
+ const union { uint32_t u; uint8_t c[4]; } one = { 1 };
28
+ return one.c[0];
29
+ }
30
+
31
+ #define PRISM_LITTLE_ENDIAN_RUNTIME pm_little_endian()
32
+ #endif
33
+
34
+ #endif
@@ -0,0 +1,17 @@
1
+ /**
2
+ * @file compiler/exceptions.hpp
3
+ */
4
+ #ifndef PRISM_COMPILER_EXCEPTIONS_HPP
5
+ #define PRISM_COMPILER_EXCEPTIONS_HPP
6
+
7
+ /**
8
+ * Defined when C++ exceptions are enabled, so that code which catches them can
9
+ * still compile in builds that disable them, such as with -fno-exceptions.
10
+ * __cpp_exceptions is the standard feature test macro, and older versions of
11
+ * GCC and MSVC define __EXCEPTIONS and _CPPUNWIND instead.
12
+ */
13
+ #if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
14
+ # define PRISM_HAS_EXCEPTIONS
15
+ #endif
16
+
17
+ #endif
@@ -4,6 +4,8 @@
4
4
  #ifndef PRISM_COMPILER_FORMAT_H
5
5
  #define PRISM_COMPILER_FORMAT_H
6
6
 
7
+ #include <stdio.h>
8
+
7
9
  /**
8
10
  * Certain compilers support specifying that a function accepts variadic
9
11
  * parameters that look like printf format strings to provide a better developer
@@ -0,0 +1,25 @@
1
+ /**
2
+ * @file compiler/lifetimebound.hpp
3
+ */
4
+ #ifndef PRISM_COMPILER_LIFETIMEBOUND_HPP
5
+ #define PRISM_COMPILER_LIFETIMEBOUND_HPP
6
+
7
+ /**
8
+ * Mark a parameter that the return value points into, so that the compiler
9
+ * warns if a temporary is passed and the return value outlives it. If it is
10
+ * placed after the qualifiers of a member function, it marks the object
11
+ * instead.
12
+ */
13
+ #if defined(__has_cpp_attribute)
14
+ # if __has_cpp_attribute(clang::lifetimebound)
15
+ # define PRISM_LIFETIMEBOUND [[clang::lifetimebound]]
16
+ # elif __has_cpp_attribute(msvc::lifetimebound)
17
+ # define PRISM_LIFETIMEBOUND [[msvc::lifetimebound]]
18
+ # endif
19
+ #endif
20
+
21
+ #ifndef PRISM_LIFETIMEBOUND
22
+ # define PRISM_LIFETIMEBOUND
23
+ #endif
24
+
25
+ #endif
@@ -11,6 +11,7 @@
11
11
  #define PRISM_CONSTANT_POOL_H
12
12
 
13
13
  #include "prism/compiler/exported.h"
14
+ #include "prism/compiler/inline.h"
14
15
  #include "prism/compiler/nodiscard.h"
15
16
  #include "prism/compiler/nonnull.h"
16
17
 
@@ -19,11 +20,31 @@
19
20
  #include <stddef.h>
20
21
  #include <stdint.h>
21
22
 
23
+ #ifdef __cplusplus
24
+ extern "C" {
25
+ #endif
26
+
22
27
  /**
23
28
  * A constant id is a unique identifier for a constant in the constant pool.
24
29
  */
25
30
  typedef uint32_t pm_constant_id_t;
26
31
 
32
+ /**
33
+ * Returns the index of the constant with the given id among the constants of
34
+ * the parser that produced it. The indices run from 0 to one less than
35
+ * pm_parser_constants_size, so they can index a cache with one entry per
36
+ * constant that was filled using pm_parser_constant_at.
37
+ *
38
+ * @param constant_id the id of the constant. Optional constant fields on nodes
39
+ * hold 0 when they are absent, which is not a valid id, so check for it
40
+ * before calling this function.
41
+ * @returns the index of the constant
42
+ */
43
+ static PRISM_INLINE size_t
44
+ pm_constant_id_index(pm_constant_id_t constant_id) {
45
+ return (size_t) constant_id - 1;
46
+ }
47
+
27
48
  /**
28
49
  * A list of constant IDs. Usually used to represent a set of locals.
29
50
  */
@@ -78,4 +99,8 @@ PRISM_EXPORTED_FUNCTION void pm_constant_id_list_init(pm_constant_id_list_t *lis
78
99
  */
79
100
  PRISM_EXPORTED_FUNCTION void pm_constant_id_list_append(pm_arena_t *arena, pm_constant_id_list_t *list, pm_constant_id_t id) PRISM_NONNULL(1, 2);
80
101
 
102
+ #ifdef __cplusplus
103
+ }
104
+ #endif
105
+
81
106
  #endif
@@ -12,6 +12,10 @@
12
12
 
13
13
  #include "prism/ast.h"
14
14
 
15
+ #ifdef __cplusplus
16
+ extern "C" {
17
+ #endif
18
+
15
19
  /**
16
20
  * An opaque pointer to a diagnostic generated during parsing.
17
21
  */
@@ -90,4 +94,18 @@ PRISM_EXPORTED_FUNCTION pm_error_level_t pm_diagnostic_error_level(const pm_diag
90
94
  */
91
95
  PRISM_EXPORTED_FUNCTION pm_warning_level_t pm_diagnostic_warning_level(const pm_diagnostic_t *diagnostic) PRISM_NONNULL(1);
92
96
 
97
+ /**
98
+ * Get the diagnostic that follows the given diagnostic among the errors or the
99
+ * warnings of the parser that produced it.
100
+ *
101
+ * @param diagnostic The diagnostic to get the successor of.
102
+ * @returns The next diagnostic of the same kind, or NULL if the given
103
+ * diagnostic is the last one.
104
+ */
105
+ PRISM_EXPORTED_FUNCTION const pm_diagnostic_t * pm_diagnostic_next(const pm_diagnostic_t *diagnostic) PRISM_NONNULL(1);
106
+
107
+ #ifdef __cplusplus
108
+ }
109
+ #endif
110
+
93
111
  #endif
@@ -14,6 +14,10 @@
14
14
  #include "prism/diagnostic.h"
15
15
  #include "prism/parser.h"
16
16
 
17
+ #ifdef __cplusplus
18
+ extern "C" {
19
+ #endif
20
+
17
21
  /** The type of formatting to use when formatting errors. */
18
22
  typedef enum {
19
23
  /** Format errors in a plain format with no colors or styles. Typically
@@ -50,4 +54,8 @@ typedef enum {
50
54
  */
51
55
  PRISM_EXPORTED_FUNCTION pm_error_level_t pm_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, pm_errors_format_type_t format_type) PRISM_NONNULL(1, 2);
52
56
 
57
+ #ifdef __cplusplus
58
+ }
59
+ #endif
60
+
53
61
  #endif
@@ -52,6 +52,9 @@ void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
52
52
  /* Append a double to the buffer. */
53
53
  void pm_buffer_append_double(pm_buffer_t *buffer, double value);
54
54
 
55
+ /* Append a 64-bit unsigned integer to the buffer in native endian order. */
56
+ void pm_buffer_append_uint64(pm_buffer_t *buffer, uint64_t value);
57
+
55
58
  /* Append a unicode codepoint to the buffer. */
56
59
  bool pm_buffer_append_unicode_codepoint(pm_buffer_t *buffer, uint32_t value);
57
60
 
@@ -46,7 +46,8 @@ typedef struct {
46
46
  /*
47
47
  * Sort the parser's comment list in place by start offset, then assign every
48
48
  * comment to the node that owns it. This is called after parsing when the
49
- * attach_comments option is set.
49
+ * attach_comments option is set, before the parser's scratch arena is released,
50
+ * because the walk allocates from it.
50
51
  */
51
52
  void pm_comments_attach(pm_parser_t *parser, const pm_node_t *root);
52
53
 
@@ -14,14 +14,18 @@
14
14
  #define PM_LOCATION_END(location_) ((location_)->start + (location_)->length)
15
15
 
16
16
  /*
17
- * Slow path for pm_node_list_append: grow the list and append the node.
18
- * Do not call directly — use pm_node_list_append instead.
17
+ * Slow path for pm_node_list_append_inline: grow the list and append the node.
18
+ * Do not call directly — use pm_node_list_append_inline instead.
19
19
  */
20
20
  void pm_node_list_append_slow(pm_arena_t *arena, pm_node_list_t *list, pm_node_t *node);
21
21
 
22
- /* Append a new node onto the end of the node list. */
22
+ /*
23
+ * Append a new node onto the end of the node list, handling the common case
24
+ * inline and growing the list through pm_node_list_append_slow. This is the
25
+ * version for prism's own code; others call the exported pm_node_list_append.
26
+ */
23
27
  static PRISM_FORCE_INLINE void
24
- pm_node_list_append(pm_arena_t *arena, pm_node_list_t *list, pm_node_t *node) {
28
+ pm_node_list_append_inline(pm_arena_t *arena, pm_node_list_t *list, pm_node_t *node) {
25
29
  if (list->size < list->capacity) {
26
30
  list->nodes[list->size++] = node;
27
31
  } else {
@@ -272,13 +272,6 @@ typedef struct pm_lex_mode {
272
272
  struct pm_lex_mode *prev;
273
273
  } pm_lex_mode_t;
274
274
 
275
- /*
276
- * We pre-allocate a certain number of lex states in order to avoid having to
277
- * call malloc too many times while parsing. You really shouldn't need more than
278
- * this because you only really nest deeply when doing string interpolation.
279
- */
280
- #define PM_LEX_STACK_SIZE 4
281
-
282
275
  /*
283
276
  * While parsing, we keep track of a stack of contexts. This is helpful for
284
277
  * error recovery so that we can pop back to a previous context when we hit a
@@ -596,9 +589,20 @@ struct pm_parser_t {
596
589
  /* The arena used for all AST-lifetime allocations. Caller-owned. */
597
590
  pm_arena_t *arena;
598
591
 
599
- /* The arena used for parser metadata (comments, diagnostics, etc.). */
592
+ /*
593
+ * The arena used for parser metadata (comments, diagnostics, etc.), which
594
+ * lives as long as the parser.
595
+ */
600
596
  pm_arena_t metadata_arena;
601
597
 
598
+ /*
599
+ * The arena used for state that is only needed while parsing, such as
600
+ * scopes, locals, and contexts. It is released when pm_parse returns, so
601
+ * nothing allocated in it can be reachable from the tree or from the
602
+ * parser's metadata.
603
+ */
604
+ pm_arena_t scratch_arena;
605
+
602
606
  /*
603
607
  * The next node identifier that will be assigned. This is a unique
604
608
  * identifier used to track nodes such that the syntax tree can be dropped
@@ -648,16 +652,23 @@ struct pm_parser_t {
648
652
  */
649
653
  pm_state_stack_t accepts_block_stack;
650
654
 
651
- /* A stack of lex modes. */
655
+ /* A stack of lex modes, linked through their prev pointers. */
652
656
  struct {
653
657
  /* The current mode of the lexer. */
654
658
  pm_lex_mode_t *current;
655
659
 
656
- /* The stack of lexer modes. */
657
- pm_lex_mode_t stack[PM_LEX_STACK_SIZE];
660
+ /*
661
+ * The mode at the bottom of the stack, which is never popped. Every
662
+ * other mode is allocated from the scratch arena.
663
+ */
664
+ pm_lex_mode_t base;
658
665
 
659
- /* The current index into the lexer mode stack. */
660
- size_t index;
666
+ /*
667
+ * Lex modes that have been popped, kept for reuse by the next push.
668
+ * They are allocated from the scratch arena, so they are never freed
669
+ * individually.
670
+ */
671
+ pm_lex_mode_t *free;
661
672
  } lex_modes;
662
673
 
663
674
  /* The pointer to the start of the source. */
@@ -715,9 +726,23 @@ struct pm_parser_t {
715
726
  /* The current local scope. */
716
727
  pm_scope_t *current_scope;
717
728
 
729
+ /*
730
+ * Scopes that have been popped, kept for reuse by the next push. Scopes
731
+ * are allocated from the scratch arena, so they are never freed
732
+ * individually.
733
+ */
734
+ pm_scope_t *free_scopes;
735
+
718
736
  /* The current parsing context. */
719
737
  pm_context_node_t *current_context;
720
738
 
739
+ /*
740
+ * Context nodes that have been popped, kept for reuse by the next push.
741
+ * Context nodes are allocated from the scratch arena, so they are never
742
+ * freed individually.
743
+ */
744
+ pm_context_node_t *free_contexts;
745
+
721
746
  /*
722
747
  * The encoding functions for the current file is attached to the parser as
723
748
  * it's parsing so that it can change with a magic comment.
@@ -981,4 +1006,10 @@ void pm_parser_init(pm_arena_t *arena, pm_parser_t *parser, const uint8_t *sourc
981
1006
  */
982
1007
  void pm_parser_cleanup(pm_parser_t *parser);
983
1008
 
1009
+ /*
1010
+ * Returns true if the bytes between start and end spell a keyword, using the
1011
+ * same lookup that the lexer does.
1012
+ */
1013
+ bool pm_keyword_p(const uint8_t *start, const uint8_t *end);
1014
+
984
1015
  #endif
@@ -1,91 +1,51 @@
1
1
  #ifndef PRISM_INTERNAL_STATIC_LITERALS_H
2
2
  #define PRISM_INTERNAL_STATIC_LITERALS_H
3
3
 
4
- #include "prism/internal/encoding.h"
5
-
6
4
  #include "prism/ast.h"
7
5
  #include "prism/buffer.h"
8
- #include "prism/line_offset_list.h"
9
-
10
- /*
11
- * An internal hash table for a set of nodes.
12
- */
13
- typedef struct {
14
- /* The array of nodes in the hash table. */
15
- pm_node_t **nodes;
16
-
17
- /* The size of the hash table. */
18
- uint32_t size;
19
-
20
- /* The space that has been allocated in the hash table. */
21
- uint32_t capacity;
22
- } pm_node_hash_t;
6
+ #include "prism/parser.h"
23
7
 
24
8
  /*
25
9
  * Certain sets of nodes (hash keys and when clauses) check for duplicate nodes
26
10
  * to alert the user of potential issues. To do this, we keep a set of the nodes
27
11
  * that have been seen so far, and compare whenever we find a new node.
28
- *
29
- * We bucket the nodes based on their type to minimize the number of comparisons
30
- * that need to be performed.
31
12
  */
32
13
  typedef struct {
33
- /*
34
- * This is the set of IntegerNode and SourceLineNode instances.
35
- */
36
- pm_node_hash_t integer_nodes;
14
+ /* The node stored in this slot, or NULL if the slot is empty. */
15
+ pm_node_t *node;
37
16
 
38
- /*
39
- * This is the set of FloatNode instances.
40
- */
41
- pm_node_hash_t float_nodes;
17
+ /* The hash of the node, kept so that it is never recomputed. */
18
+ uint64_t hash;
19
+ } pm_static_literals_entry_t;
42
20
 
43
- /*
44
- * This is the set of RationalNode and ImaginaryNode instances.
45
- */
46
- pm_node_hash_t number_nodes;
47
-
48
- /*
49
- * This is the set of StringNode and SourceFileNode instances.
50
- */
51
- pm_node_hash_t string_nodes;
52
-
53
- /*
54
- * This is the set of RegularExpressionNode instances.
55
- */
56
- pm_node_hash_t regexp_nodes;
21
+ /*
22
+ * The number of nodes a set holds directly before it moves them into a hash
23
+ * table. Most sets are this small, so they are searched linearly without
24
+ * hashing their nodes or allocating any memory.
25
+ */
26
+ #define PM_STATIC_LITERALS_INLINE_CAPACITY 4
57
27
 
28
+ typedef struct {
58
29
  /*
59
- * This is the set of SymbolNode instances.
30
+ * The nodes in the set while it has no hash table. Once the set outgrows
31
+ * this array, its nodes move into the hash table and this is unused.
60
32
  */
61
- pm_node_hash_t symbol_nodes;
33
+ pm_node_t *nodes[PM_STATIC_LITERALS_INLINE_CAPACITY];
62
34
 
63
- /*
64
- * A pointer to the last TrueNode instance that was inserted, or NULL.
65
- */
66
- pm_node_t *true_node;
35
+ /* The array of entries in the hash table, or NULL if there is none. */
36
+ pm_static_literals_entry_t *entries;
67
37
 
68
- /*
69
- * A pointer to the last FalseNode instance that was inserted, or NULL.
70
- */
71
- pm_node_t *false_node;
72
-
73
- /*
74
- * A pointer to the last NilNode instance that was inserted, or NULL.
75
- */
76
- pm_node_t *nil_node;
38
+ /* The number of nodes in the set. */
39
+ uint32_t size;
77
40
 
78
- /*
79
- * A pointer to the last SourceEncodingNode instance that was inserted, or
80
- * NULL.
81
- */
82
- pm_node_t *source_encoding_node;
41
+ /* The number of entries allocated in the hash table, or 0 if there is none. */
42
+ uint32_t capacity;
83
43
  } pm_static_literals_t;
84
44
 
85
45
  /*
86
46
  * Add a node to the set of static literals.
87
47
  */
88
- pm_node_t * pm_static_literals_add(const pm_line_offset_list_t *line_offsets, const uint8_t *start, int32_t start_line, const pm_encoding_t *encoding, pm_static_literals_t *literals, pm_node_t *node, bool replace);
48
+ pm_node_t * pm_static_literals_add(const pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node, bool replace);
89
49
 
90
50
  /*
91
51
  * Free the internal memory associated with the given static literals set.
@@ -95,6 +55,6 @@ void pm_static_literals_free(pm_static_literals_t *literals);
95
55
  /*
96
56
  * Create a string-based representation of the given static literal.
97
57
  */
98
- void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_line_offset_list_t *line_offsets, const uint8_t *start, int32_t start_line, const pm_encoding_t *encoding, const pm_node_t *node);
58
+ void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node);
99
59
 
100
60
  #endif