prism 1.9.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +144 -40
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +49 -20
  6. data/README.md +38 -98
  7. data/config.yml +174 -161
  8. data/docs/build_system.md +2 -2
  9. data/docs/configuration.md +5 -4
  10. data/docs/encoding.md +2 -2
  11. data/docs/fuzzing.md +9 -16
  12. data/docs/javascript.md +19 -4
  13. data/docs/mapping.md +1 -1
  14. data/docs/releasing.md +77 -5
  15. data/docs/relocation.md +2 -4
  16. data/docs/serialization.md +39 -28
  17. data/docs/testing.md +46 -29
  18. data/ext/prism/api_node.c +1527 -1467
  19. data/ext/prism/extconf.rb +3 -4
  20. data/ext/prism/extension.c +759 -418
  21. data/ext/prism/extension.h +3 -3
  22. data/include/prism/arena.h +45 -0
  23. data/include/prism/arena.hpp +42 -0
  24. data/include/prism/ast.h +2738 -611
  25. data/include/prism/ast.hpp +641 -0
  26. data/include/prism/buffer.h +60 -0
  27. data/include/prism/buffer.hpp +55 -0
  28. data/include/prism/comments.h +72 -0
  29. data/include/prism/compiler/accel.h +26 -0
  30. data/include/prism/compiler/align.h +36 -0
  31. data/include/prism/compiler/assume.h +25 -0
  32. data/include/prism/compiler/branch.h +26 -0
  33. data/include/prism/compiler/bswap.h +46 -0
  34. data/include/prism/compiler/cplusplus.hpp +16 -0
  35. data/include/prism/compiler/endianness.h +34 -0
  36. data/include/prism/compiler/exceptions.hpp +17 -0
  37. data/include/prism/compiler/exported.h +24 -0
  38. data/include/prism/compiler/fallthrough.h +22 -0
  39. data/include/prism/compiler/filesystem.h +32 -0
  40. data/include/prism/compiler/flex_array.h +19 -0
  41. data/include/prism/compiler/force_inline.h +21 -0
  42. data/include/prism/compiler/format.h +27 -0
  43. data/include/prism/compiler/inline.h +17 -0
  44. data/include/prism/compiler/lifetimebound.hpp +25 -0
  45. data/include/prism/compiler/nodiscard.h +22 -0
  46. data/include/prism/compiler/nonnull.h +18 -0
  47. data/include/prism/compiler/unused.h +18 -0
  48. data/include/prism/constant_pool.h +106 -0
  49. data/include/prism/diagnostic.h +55 -402
  50. data/include/prism/errors_format.h +61 -0
  51. data/include/prism/excludes.h +29 -0
  52. data/include/prism/integer.h +41 -0
  53. data/include/prism/internal/allocator.h +68 -0
  54. data/include/prism/internal/allocator_debug.h +88 -0
  55. data/include/prism/internal/arena.h +108 -0
  56. data/include/prism/internal/bit.h +42 -0
  57. data/include/prism/internal/buffer.h +94 -0
  58. data/include/prism/internal/char.h +145 -0
  59. data/include/prism/internal/comments.h +54 -0
  60. data/include/prism/internal/constant_pool.h +152 -0
  61. data/include/prism/internal/diagnostic.h +389 -0
  62. data/include/prism/{encoding.h → internal/encoding.h} +27 -68
  63. data/include/prism/internal/integer.h +68 -0
  64. data/include/prism/internal/isinf.h +16 -0
  65. data/include/prism/internal/line_offset_list.h +34 -0
  66. data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  67. data/include/prism/internal/magic_comments.h +23 -0
  68. data/include/prism/internal/memchr.h +15 -0
  69. data/include/prism/internal/node.h +42 -0
  70. data/include/prism/internal/options.h +232 -0
  71. data/include/prism/internal/parser.h +1015 -0
  72. data/include/prism/internal/regexp.h +26 -0
  73. data/include/prism/internal/serialize.h +39 -0
  74. data/include/prism/internal/source.h +72 -0
  75. data/include/prism/internal/static_literals.h +60 -0
  76. data/include/prism/internal/stringy.h +30 -0
  77. data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  78. data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  79. data/include/prism/internal/tokens.h +11 -0
  80. data/include/prism/internal/xxh3.h +16 -0
  81. data/include/prism/json.h +40 -0
  82. data/include/prism/line_offset_list.h +69 -0
  83. data/include/prism/magic_comments.h +53 -0
  84. data/include/prism/node.h +59 -49
  85. data/include/prism/node.hpp +272 -0
  86. data/include/prism/node_type.hpp +38 -0
  87. data/include/prism/options.h +154 -279
  88. data/include/prism/options.hpp +43 -0
  89. data/include/prism/parser.h +367 -870
  90. data/include/prism/parser.hpp +274 -0
  91. data/include/prism/prettyprint.h +14 -9
  92. data/include/prism/serialize.h +123 -0
  93. data/include/prism/source.h +163 -0
  94. data/include/prism/stream.h +36 -0
  95. data/include/prism/string_query.h +82 -0
  96. data/include/prism/stringy.h +61 -0
  97. data/include/prism/stringy.hpp +31 -0
  98. data/include/prism/version.h +20 -3
  99. data/include/prism.h +35 -320
  100. data/include/prism.hpp +24 -0
  101. data/lib/prism/compiler.rb +319 -305
  102. data/lib/prism/desugar_compiler.rb +105 -34
  103. data/lib/prism/dispatcher.rb +646 -473
  104. data/lib/prism/dot_visitor.rb +592 -511
  105. data/lib/prism/dsl.rb +391 -25
  106. data/lib/prism/ffi.rb +258 -90
  107. data/lib/prism/inspect_visitor.rb +357 -325
  108. data/lib/prism/lex_compat.rb +254 -212
  109. data/lib/prism/mutation_compiler.rb +313 -305
  110. data/lib/prism/node.rb +20966 -6992
  111. data/lib/prism/node_ext.rb +180 -187
  112. data/lib/prism/node_find.rb +215 -0
  113. data/lib/prism/parse_result/errors.rb +7 -1
  114. data/lib/prism/parse_result/newlines.rb +276 -8
  115. data/lib/prism/parse_result.rb +468 -139
  116. data/lib/prism/pattern.rb +65 -20
  117. data/lib/prism/reflection.rb +23 -12
  118. data/lib/prism/relocation.rb +149 -87
  119. data/lib/prism/serialize.rb +2889 -700
  120. data/lib/prism/string_query.rb +24 -1
  121. data/lib/prism/translation/parser/builder.rb +10 -2
  122. data/lib/prism/translation/parser/compiler.rb +44 -52
  123. data/lib/prism/translation/parser/lexer.rb +41 -92
  124. data/lib/prism/translation/parser.rb +1 -1
  125. data/lib/prism/translation/parser_current.rb +1 -1
  126. data/lib/prism/translation/ripper/lexer.rb +8 -10
  127. data/lib/prism/translation/ripper/sexp.rb +2 -10
  128. data/lib/prism/translation/ripper/shim.rb +2 -0
  129. data/lib/prism/translation/ripper.rb +967 -209
  130. data/lib/prism/translation/ruby_parser.rb +73 -316
  131. data/lib/prism/translation.rb +2 -0
  132. data/lib/prism/visitor.rb +632 -5
  133. data/lib/prism.rb +79 -9
  134. data/prism.gemspec +141 -62
  135. data/rbi/generated/prism/compiler.rbi +489 -0
  136. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  137. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  138. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  139. data/rbi/generated/prism/dsl.rbi +751 -0
  140. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  141. data/rbi/generated/prism/lex_compat.rbi +162 -0
  142. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  143. data/rbi/generated/prism/node.rbi +15143 -0
  144. data/rbi/generated/prism/node_ext.rbi +259 -0
  145. data/rbi/generated/prism/node_find.rbi +79 -0
  146. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  147. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  148. data/rbi/generated/prism/parse_result.rbi +776 -0
  149. data/rbi/generated/prism/pattern.rbi +134 -0
  150. data/rbi/generated/prism/reflection.rbi +89 -0
  151. data/rbi/generated/prism/relocation.rbi +329 -0
  152. data/rbi/generated/prism/serialize.rbi +185 -0
  153. data/rbi/generated/prism/string_query.rbi +43 -0
  154. data/rbi/generated/prism/translation.rbi +7 -0
  155. data/rbi/generated/prism/visitor.rbi +650 -0
  156. data/rbi/generated/prism.rbi +86 -0
  157. data/rbi/prism/translation/parser_versions.rbi +5 -0
  158. data/rbi/rubyvm/node_find.rbi +14 -0
  159. data/sig/generated/prism/compiler.rbs +492 -0
  160. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  161. data/sig/generated/prism/dispatcher.rbs +1292 -0
  162. data/sig/generated/prism/dot_visitor.rbs +635 -0
  163. data/sig/generated/prism/dsl.rbs +928 -0
  164. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  165. data/sig/generated/prism/lex_compat.rbs +166 -0
  166. data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
  167. data/sig/generated/prism/node.rbs +21632 -0
  168. data/sig/generated/prism/node_ext.rbs +290 -0
  169. data/sig/generated/prism/node_find.rbs +90 -0
  170. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  171. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  172. data/sig/generated/prism/parse_result.rbs +871 -0
  173. data/sig/generated/prism/pattern.rbs +156 -0
  174. data/sig/generated/prism/reflection.rbs +101 -0
  175. data/sig/generated/prism/relocation.rbs +428 -0
  176. data/sig/generated/prism/serialize.rbs +200 -0
  177. data/sig/generated/prism/string_query.rbs +43 -0
  178. data/sig/generated/prism/translation.rbs +8 -0
  179. data/sig/generated/prism/visitor.rbs +1111 -0
  180. data/sig/generated/prism.rbs +84 -0
  181. data/sig/manifest.yaml +2 -0
  182. data/src/arena.c +122 -0
  183. data/src/{util/pm_buffer.c → buffer.c} +53 -21
  184. data/src/{util/pm_char.c → char.c} +28 -57
  185. data/src/comments.c +455 -0
  186. data/src/constant_pool.c +406 -0
  187. data/src/diagnostic.c +120 -68
  188. data/src/encoding.c +8 -3
  189. data/src/errors_format.c +519 -0
  190. data/src/{util/pm_integer.c → integer.c} +38 -22
  191. data/src/json.c +5733 -0
  192. data/src/line_offset_list.c +100 -0
  193. data/src/list.c +24 -0
  194. data/src/{util/pm_memchr.c → memchr.c} +8 -6
  195. data/src/node.c +3465 -7787
  196. data/src/options.c +161 -54
  197. data/src/parser.c +371 -0
  198. data/src/prettyprint.c +421 -354
  199. data/src/prism.c +8503 -7280
  200. data/src/regexp.c +960 -45
  201. data/src/serialize.c +2434 -2048
  202. data/src/source.c +524 -0
  203. data/src/static_literals.c +303 -360
  204. data/src/string_query.c +180 -0
  205. data/src/stringy.c +61 -0
  206. data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  207. data/src/strpbrk.c +447 -0
  208. data/src/{token_type.c → tokens.c} +39 -18
  209. data/src/xxh3.c +292 -0
  210. metadata +141 -63
  211. data/ext/prism/api_pack.c +0 -276
  212. data/include/prism/defines.h +0 -260
  213. data/include/prism/pack.h +0 -163
  214. data/include/prism/regexp.h +0 -43
  215. data/include/prism/static_literals.h +0 -121
  216. data/include/prism/util/pm_buffer.h +0 -236
  217. data/include/prism/util/pm_char.h +0 -204
  218. data/include/prism/util/pm_constant_pool.h +0 -218
  219. data/include/prism/util/pm_integer.h +0 -130
  220. data/include/prism/util/pm_memchr.h +0 -29
  221. data/include/prism/util/pm_newline_list.h +0 -113
  222. data/include/prism/util/pm_string.h +0 -200
  223. data/lib/prism/pack.rb +0 -230
  224. data/lib/prism/parse_result/comments.rb +0 -188
  225. data/rbi/prism/compiler.rbi +0 -12
  226. data/rbi/prism/dsl.rbi +0 -524
  227. data/rbi/prism/inspect_visitor.rbi +0 -12
  228. data/rbi/prism/node.rbi +0 -8750
  229. data/rbi/prism/node_ext.rbi +0 -107
  230. data/rbi/prism/parse_result.rbi +0 -404
  231. data/rbi/prism/reflection.rbi +0 -58
  232. data/rbi/prism/string_query.rbi +0 -12
  233. data/rbi/prism/visitor.rbi +0 -473
  234. data/rbi/prism.rbi +0 -63
  235. data/sig/prism/compiler.rbs +0 -9
  236. data/sig/prism/dispatcher.rbs +0 -19
  237. data/sig/prism/dot_visitor.rbs +0 -6
  238. data/sig/prism/dsl.rbs +0 -351
  239. data/sig/prism/inspect_visitor.rbs +0 -22
  240. data/sig/prism/lex_compat.rbs +0 -10
  241. data/sig/prism/node.rbs +0 -4042
  242. data/sig/prism/node_ext.rbs +0 -149
  243. data/sig/prism/pack.rbs +0 -43
  244. data/sig/prism/parse_result/comments.rbs +0 -38
  245. data/sig/prism/parse_result.rbs +0 -197
  246. data/sig/prism/pattern.rbs +0 -13
  247. data/sig/prism/reflection.rbs +0 -50
  248. data/sig/prism/relocation.rbs +0 -185
  249. data/sig/prism/serialize.rbs +0 -8
  250. data/sig/prism/string_query.rbs +0 -11
  251. data/sig/prism/visitor.rbs +0 -169
  252. data/sig/prism.rbs +0 -272
  253. data/src/pack.c +0 -509
  254. data/src/util/pm_constant_pool.c +0 -342
  255. data/src/util/pm_list.c +0 -49
  256. data/src/util/pm_newline_list.c +0 -125
  257. data/src/util/pm_string.c +0 -381
  258. data/src/util/pm_strpbrk.c +0 -206
data/src/xxh3.c ADDED
@@ -0,0 +1,292 @@
1
+ /**
2
+ * An implementation of the XXH3 hash function.
3
+ *
4
+ * The reason some of these names do not look quite like the style of the rest
5
+ * of the code in Prism is that this was adapted straight from the XXH3
6
+ * specification here:
7
+ *
8
+ * https://github.com/Cyan4973/xxHash/blob/release/doc/xxhash_spec.md
9
+ *
10
+ * To better understand this code it is recommended to read the XXH3
11
+ * specification linked above, where you should be able to follow along with
12
+ * this implementation more easily. Note this only implements the XXH3_64
13
+ * variant with the default secret and a seed of 0. Additionally it does not
14
+ * include a lot of the various CPU-specific optimizations that exist in the
15
+ * reference implementation.
16
+ */
17
+
18
+ #include "prism/internal/xxh3.h"
19
+
20
+ #include "prism/compiler/accel.h"
21
+ #include "prism/compiler/bswap.h"
22
+ #include "prism/compiler/endianness.h"
23
+
24
+ #include <string.h>
25
+
26
+ #if defined(PRISM_HAS_NEON) && defined(PRISM_LITTLE_ENDIAN_COMPTIME)
27
+ #include <arm_neon.h>
28
+ #define XXH3_NEON
29
+ #elif defined(PRISM_HAS_SSE2)
30
+ #include <emmintrin.h>
31
+ #define XXH3_SSE2
32
+ #endif
33
+
34
+ #if defined(__SIZEOF_INT128__)
35
+ static uint64_t
36
+ XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
37
+ __uint128_t x = (__uint128_t) lhs * (__uint128_t) rhs;
38
+ return ((uint64_t) (x & 0xFFFFFFFFFFFFFFFFULL)) ^ ((uint64_t) (x >> 64));
39
+ }
40
+ #else
41
+ #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64))
42
+ #include <intrin.h>
43
+ #endif
44
+
45
+ static uint64_t
46
+ XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
47
+ uint64_t lo, hi;
48
+ #if defined(_MSC_VER) && defined(_M_X64)
49
+ lo = _umul128(lhs, rhs, &hi);
50
+ #elif defined(_MSC_VER) && defined(_M_ARM64)
51
+ hi = __umulh(lhs, rhs);
52
+ lo = lhs * rhs;
53
+ #else
54
+ uint64_t lo_lo = (lhs & 0xFFFFFFFF) * (rhs & 0xFFFFFFFF);
55
+ uint64_t hi_lo = (lhs >> 32) * (rhs & 0xFFFFFFFF);
56
+ uint64_t lo_hi = (lhs & 0xFFFFFFFF) * (rhs >> 32);
57
+ uint64_t hi_hi = (lhs >> 32) * (rhs >> 32);
58
+
59
+ uint64_t cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
60
+ hi = hi_hi + (hi_lo >> 32) + (cross >> 32);
61
+ lo = (cross << 32) | (lo_lo & 0xFFFFFFFF);
62
+ #endif
63
+ return lo ^ hi;
64
+ }
65
+ #endif
66
+
67
+ static uint32_t
68
+ read32(const void* ptr) {
69
+ uint32_t val;
70
+ memcpy(&val, ptr, sizeof(val));
71
+ if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
72
+ val = pm_bswap32(val);
73
+ }
74
+ return val;
75
+ }
76
+
77
+ static uint64_t
78
+ read64(const void* ptr) {
79
+ uint64_t val;
80
+ memcpy(&val, ptr, sizeof(val));
81
+ if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
82
+ val = pm_bswap64(val);
83
+ }
84
+ return val;
85
+ }
86
+
87
+ static const uint64_t PRIME32_1 = 0x9E3779B1U;
88
+ static const uint64_t PRIME32_2 = 0x85EBCA77U;
89
+ static const uint64_t PRIME32_3 = 0xC2B2AE3DU;
90
+ static const uint64_t PRIME64_1 = 0x9E3779B185EBCA87ULL;
91
+ static const uint64_t PRIME64_2 = 0xC2B2AE3D27D4EB4FULL;
92
+ static const uint64_t PRIME64_3 = 0x165667B19E3779F9ULL;
93
+ static const uint64_t PRIME64_4 = 0x85EBCA77C2B2AE63ULL;
94
+ static const uint64_t PRIME64_5 = 0x27D4EB2F165667C5ULL;
95
+ static const uint64_t PRIME_MX1 = 0x165667919E3779F9ULL;
96
+ static const uint64_t PRIME_MX2 = 0x9FB21C651E98DF25ULL;
97
+
98
+ static const uint8_t secret[192] = {
99
+ 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
100
+ 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
101
+ 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
102
+ 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
103
+ 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
104
+ 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
105
+ 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
106
+ 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
107
+ 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
108
+ 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
109
+ 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
110
+ 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
111
+ };
112
+
113
+ static uint64_t
114
+ avalanche(uint64_t x) {
115
+ x ^= (x >> 37);
116
+ x *= PRIME_MX1;
117
+ x ^= (x >> 32);
118
+ return x;
119
+ }
120
+
121
+ static uint64_t
122
+ avalanche_XXH64(uint64_t x) {
123
+ x ^= (x >> 33);
124
+ x *= PRIME64_2;
125
+ x ^= (x >> 29);
126
+ x *= PRIME64_3;
127
+ x ^= (x >> 32);
128
+ return x;
129
+ }
130
+
131
+ static uint64_t
132
+ XXH3_64_empty(void) {
133
+ return avalanche_XXH64(read64(secret + 56) ^ read64(secret + 64));
134
+ }
135
+
136
+ static uint64_t
137
+ XXH3_64_1to3(const uint8_t *input, size_t length) {
138
+ uint32_t combined = (uint32_t) input[length - 1] | ((uint32_t) length << 8) | ((uint32_t) input[0] << 16) | ((uint32_t) input[length >> 1] << 24);
139
+ return avalanche_XXH64((uint64_t) (read32(secret) ^ read32(secret + 4)) ^ (uint64_t) combined);
140
+ }
141
+
142
+ static uint64_t
143
+ XXH3_64_4to8(const uint8_t *input, size_t length) {
144
+ #define rot(x_, r_) (((x_) << (r_)) | ((x_) >> (64 - (r_))))
145
+
146
+ uint64_t combined = (uint64_t) read32(input + length - 4) | ((uint64_t) read32(input) << 32);
147
+ uint64_t value = (read64(secret + 8) ^ read64(secret + 16)) ^ combined;
148
+ value ^= rot(value, 49) ^ rot(value, 24);
149
+ value *= PRIME_MX2;
150
+ value ^= ((value >> 35) + length);
151
+ value *= PRIME_MX2;
152
+ value ^= (value >> 28);
153
+ return value;
154
+
155
+ #undef rot
156
+ }
157
+
158
+ static uint64_t
159
+ XXH3_64_9to16(const uint8_t *input, size_t length) {
160
+ uint64_t lo = (read64(secret + 24) ^ read64(secret + 32)) ^ read64(input);
161
+ uint64_t hi = (read64(secret + 40) ^ read64(secret + 48)) ^ read64(input + length - 8);
162
+ return avalanche(length + pm_bswap64(lo) + hi + XXH3_mul128_fold64(lo, hi));
163
+ }
164
+
165
+ static uint64_t
166
+ XXH3_64_17to240(const uint8_t *input, size_t length) {
167
+ #define mix(data_, offset_) XXH3_mul128_fold64(read64((data_)) ^ read64(secret + (offset_)), read64((data_) + 8) ^ read64(secret + (offset_) + 8))
168
+
169
+ uint64_t acc = length * PRIME64_1;
170
+
171
+ if (length <= 128) {
172
+ size_t idx = (length - 1) / 32;
173
+ do {
174
+ acc += mix(input + idx * 16, idx * 32) + mix(input + length - idx * 16 - 16, idx * 32 + 16);
175
+ } while (idx--);
176
+ } else {
177
+ for (size_t idx = 0; idx < 8; idx++) {
178
+ acc += mix(input + idx * 16, idx * 16);
179
+ }
180
+ acc = avalanche(acc);
181
+ for (size_t idx = 8; idx < length >> 4; idx++) {
182
+ acc += mix(input + idx * 16, (idx - 8) * 16 + 3);
183
+ }
184
+ acc += mix(input + length - 16, 119);
185
+ }
186
+
187
+ return avalanche(acc);
188
+
189
+ #undef mix
190
+ }
191
+
192
+ static void
193
+ accumulate(uint64_t acc[8], const uint8_t *stripe, size_t secret_offset) {
194
+ #if defined(XXH3_NEON)
195
+ for (int idx = 0; idx < 8; idx += 2) {
196
+ uint64x2_t data = vreinterpretq_u64_u8(vld1q_u8(stripe + 8 * idx));
197
+ uint64x2_t value = veorq_u64(data, vreinterpretq_u64_u8(vld1q_u8(secret + secret_offset + 8 * idx)));
198
+ uint64x2_t product = vmull_u32(vmovn_u64(value), vshrn_n_u64(value, 32));
199
+ vst1q_u64(acc + idx, vaddq_u64(vld1q_u64(acc + idx), vaddq_u64(vextq_u64(data, data, 1), product)));
200
+ }
201
+ #elif defined(XXH3_SSE2)
202
+ for (int idx = 0; idx < 8; idx += 2) {
203
+ __m128i data = _mm_loadu_si128((const __m128i *) (stripe + 8 * idx));
204
+ __m128i value = _mm_xor_si128(data, _mm_loadu_si128((const __m128i *) (secret + secret_offset + 8 * idx)));
205
+ __m128i product = _mm_mul_epu32(value, _mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)));
206
+ _mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_loadu_si128((const __m128i *) (acc + idx)), _mm_add_epi64(_mm_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2)), product)));
207
+ }
208
+ #else
209
+ for (int idx = 0; idx < 8; idx++) {
210
+ uint64_t data = read64(stripe + 8 * idx);
211
+ uint64_t value = data ^ read64(secret + secret_offset + 8 * idx);
212
+ acc[idx ^ 1] = acc[idx ^ 1] + data;
213
+ acc[idx] += (value & 0xFFFFFFFF) * (value >> 32);
214
+ }
215
+ #endif
216
+ }
217
+
218
+ static uint64_t
219
+ XXH3_64_large(const uint8_t *input, size_t length) {
220
+ const size_t secret_length = sizeof(secret);
221
+ const size_t stripes_per_block = (secret_length - 64) / 8;
222
+ const size_t block_size = 64 * stripes_per_block;
223
+ const size_t block_count = (length - 1) / block_size;
224
+ const size_t last_offset = block_count * block_size;
225
+
226
+ uint64_t acc[8] = {
227
+ PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3,
228
+ PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1
229
+ };
230
+
231
+ for (size_t block_idx = 0; block_idx < block_count; block_idx++) {
232
+ const uint8_t *block = input + block_idx * block_size;
233
+
234
+ for (size_t stripe_idx = 0; stripe_idx < stripes_per_block; stripe_idx++) {
235
+ accumulate(acc, block + stripe_idx * 64, stripe_idx * 8);
236
+ }
237
+
238
+ #if defined(XXH3_NEON)
239
+ const uint32x2_t prime = vdup_n_u32((uint32_t) PRIME32_1);
240
+ for (size_t idx = 0; idx < 8; idx += 2) {
241
+ uint64x2_t value = vld1q_u64(acc + idx);
242
+ value = veorq_u64(value, vshrq_n_u64(value, 47));
243
+ value = veorq_u64(value, vreinterpretq_u64_u8(vld1q_u8(secret + secret_length - 64 + idx * 8)));
244
+ vst1q_u64(acc + idx, vaddq_u64(vmull_u32(vmovn_u64(value), prime), vshlq_n_u64(vmull_u32(vshrn_n_u64(value, 32), prime), 32)));
245
+ }
246
+ #elif defined(XXH3_SSE2)
247
+ const __m128i prime = _mm_set1_epi32((int) (uint32_t) PRIME32_1);
248
+ for (size_t idx = 0; idx < 8; idx += 2) {
249
+ __m128i value = _mm_loadu_si128((const __m128i *) (acc + idx));
250
+ value = _mm_xor_si128(value, _mm_srli_epi64(value, 47));
251
+ value = _mm_xor_si128(value, _mm_loadu_si128((const __m128i *) (secret + secret_length - 64 + idx * 8)));
252
+ _mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_mul_epu32(value, prime), _mm_slli_epi64(_mm_mul_epu32(_mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)), prime), 32)));
253
+ }
254
+ #else
255
+ for (size_t idx = 0; idx < 8; idx++) {
256
+ acc[idx] ^= (acc[idx] >> 47);
257
+ acc[idx] ^= read64(secret + secret_length - 64 + idx * 8);
258
+ acc[idx] *= PRIME32_1;
259
+ }
260
+ #endif
261
+ }
262
+
263
+ for (size_t stripe_idx = 0; stripe_idx < (length - last_offset - 1) / 64; stripe_idx++) {
264
+ accumulate(acc, input + last_offset + stripe_idx * 64, stripe_idx * 8);
265
+ }
266
+
267
+ accumulate(acc, input + length - 64, secret_length - 71);
268
+
269
+ uint64_t result = (uint64_t) length * PRIME64_1;
270
+ for (size_t idx = 0; idx < 4; idx++) {
271
+ result += XXH3_mul128_fold64(acc[idx * 2] ^ read64(secret + 11 + 16 * idx), acc[idx * 2 + 1] ^ read64(secret + 11 + 16 * idx + 8));
272
+ }
273
+
274
+ return avalanche(result);
275
+ }
276
+
277
+ uint64_t
278
+ pm_xxh3(const uint8_t *input, size_t length) {
279
+ if (length == 0) {
280
+ return XXH3_64_empty();
281
+ } else if (length <= 3) {
282
+ return XXH3_64_1to3(input, length);
283
+ } else if (length <= 8) {
284
+ return XXH3_64_4to8(input, length);
285
+ } else if (length <= 16) {
286
+ return XXH3_64_9to16(input, length);
287
+ } else if (length <= 240) {
288
+ return XXH3_64_17to240(input, length);
289
+ } else {
290
+ return XXH3_64_large(input, length);
291
+ }
292
+ }
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: prism
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.9.0
4
+ version: 2.1.0
5
5
  platform: ruby
6
6
  authors:
7
- - Shopify
7
+ - Ruby developers
8
8
  bindir: bin
9
9
  cert_chain: []
10
10
  date: 1980-01-02 00:00:00.000000000 Z
11
11
  dependencies: []
12
12
  email:
13
- - ruby@shopify.com
13
+ - ruby-core@ruby-lang.org
14
14
  executables: []
15
15
  extensions:
16
16
  - ext/prism/extconf.rb
@@ -44,32 +44,87 @@ files:
44
44
  - docs/serialization.md
45
45
  - docs/testing.md
46
46
  - ext/prism/api_node.c
47
- - ext/prism/api_pack.c
48
47
  - ext/prism/extconf.rb
49
48
  - ext/prism/extension.c
50
49
  - ext/prism/extension.h
51
50
  - include/prism.h
51
+ - include/prism.hpp
52
+ - include/prism/arena.h
53
+ - include/prism/arena.hpp
52
54
  - include/prism/ast.h
53
- - include/prism/defines.h
55
+ - include/prism/ast.hpp
56
+ - include/prism/buffer.h
57
+ - include/prism/buffer.hpp
58
+ - include/prism/comments.h
59
+ - include/prism/compiler/accel.h
60
+ - include/prism/compiler/align.h
61
+ - include/prism/compiler/assume.h
62
+ - include/prism/compiler/branch.h
63
+ - include/prism/compiler/bswap.h
64
+ - include/prism/compiler/cplusplus.hpp
65
+ - include/prism/compiler/endianness.h
66
+ - include/prism/compiler/exceptions.hpp
67
+ - include/prism/compiler/exported.h
68
+ - include/prism/compiler/fallthrough.h
69
+ - include/prism/compiler/filesystem.h
70
+ - include/prism/compiler/flex_array.h
71
+ - include/prism/compiler/force_inline.h
72
+ - include/prism/compiler/format.h
73
+ - include/prism/compiler/inline.h
74
+ - include/prism/compiler/lifetimebound.hpp
75
+ - include/prism/compiler/nodiscard.h
76
+ - include/prism/compiler/nonnull.h
77
+ - include/prism/compiler/unused.h
78
+ - include/prism/constant_pool.h
54
79
  - include/prism/diagnostic.h
55
- - include/prism/encoding.h
80
+ - include/prism/errors_format.h
81
+ - include/prism/excludes.h
82
+ - include/prism/integer.h
83
+ - include/prism/internal/allocator.h
84
+ - include/prism/internal/allocator_debug.h
85
+ - include/prism/internal/arena.h
86
+ - include/prism/internal/bit.h
87
+ - include/prism/internal/buffer.h
88
+ - include/prism/internal/char.h
89
+ - include/prism/internal/comments.h
90
+ - include/prism/internal/constant_pool.h
91
+ - include/prism/internal/diagnostic.h
92
+ - include/prism/internal/encoding.h
93
+ - include/prism/internal/integer.h
94
+ - include/prism/internal/isinf.h
95
+ - include/prism/internal/line_offset_list.h
96
+ - include/prism/internal/list.h
97
+ - include/prism/internal/magic_comments.h
98
+ - include/prism/internal/memchr.h
99
+ - include/prism/internal/node.h
100
+ - include/prism/internal/options.h
101
+ - include/prism/internal/parser.h
102
+ - include/prism/internal/regexp.h
103
+ - include/prism/internal/serialize.h
104
+ - include/prism/internal/source.h
105
+ - include/prism/internal/static_literals.h
106
+ - include/prism/internal/stringy.h
107
+ - include/prism/internal/strncasecmp.h
108
+ - include/prism/internal/strpbrk.h
109
+ - include/prism/internal/tokens.h
110
+ - include/prism/internal/xxh3.h
111
+ - include/prism/json.h
112
+ - include/prism/line_offset_list.h
113
+ - include/prism/magic_comments.h
56
114
  - include/prism/node.h
115
+ - include/prism/node.hpp
116
+ - include/prism/node_type.hpp
57
117
  - include/prism/options.h
58
- - include/prism/pack.h
118
+ - include/prism/options.hpp
59
119
  - include/prism/parser.h
120
+ - include/prism/parser.hpp
60
121
  - include/prism/prettyprint.h
61
- - include/prism/regexp.h
62
- - include/prism/static_literals.h
63
- - include/prism/util/pm_buffer.h
64
- - include/prism/util/pm_char.h
65
- - include/prism/util/pm_constant_pool.h
66
- - include/prism/util/pm_integer.h
67
- - include/prism/util/pm_list.h
68
- - include/prism/util/pm_memchr.h
69
- - include/prism/util/pm_newline_list.h
70
- - include/prism/util/pm_string.h
71
- - include/prism/util/pm_strncasecmp.h
72
- - include/prism/util/pm_strpbrk.h
122
+ - include/prism/serialize.h
123
+ - include/prism/source.h
124
+ - include/prism/stream.h
125
+ - include/prism/string_query.h
126
+ - include/prism/stringy.h
127
+ - include/prism/stringy.hpp
73
128
  - include/prism/version.h
74
129
  - lib/prism.rb
75
130
  - lib/prism/compiler.rb
@@ -83,9 +138,8 @@ files:
83
138
  - lib/prism/mutation_compiler.rb
84
139
  - lib/prism/node.rb
85
140
  - lib/prism/node_ext.rb
86
- - lib/prism/pack.rb
141
+ - lib/prism/node_find.rb
87
142
  - lib/prism/parse_result.rb
88
- - lib/prism/parse_result/comments.rb
89
143
  - lib/prism/parse_result/errors.rb
90
144
  - lib/prism/parse_result/newlines.rb
91
145
  - lib/prism/pattern.rb
@@ -113,59 +167,83 @@ files:
113
167
  - lib/prism/translation/ruby_parser.rb
114
168
  - lib/prism/visitor.rb
115
169
  - prism.gemspec
116
- - rbi/prism.rbi
117
- - rbi/prism/compiler.rbi
118
- - rbi/prism/dsl.rbi
119
- - rbi/prism/inspect_visitor.rbi
120
- - rbi/prism/node.rbi
121
- - rbi/prism/node_ext.rbi
122
- - rbi/prism/parse_result.rbi
123
- - rbi/prism/reflection.rbi
124
- - rbi/prism/string_query.rbi
170
+ - rbi/generated/prism.rbi
171
+ - rbi/generated/prism/compiler.rbi
172
+ - rbi/generated/prism/desugar_compiler.rbi
173
+ - rbi/generated/prism/dispatcher.rbi
174
+ - rbi/generated/prism/dot_visitor.rbi
175
+ - rbi/generated/prism/dsl.rbi
176
+ - rbi/generated/prism/inspect_visitor.rbi
177
+ - rbi/generated/prism/lex_compat.rbi
178
+ - rbi/generated/prism/mutation_compiler.rbi
179
+ - rbi/generated/prism/node.rbi
180
+ - rbi/generated/prism/node_ext.rbi
181
+ - rbi/generated/prism/node_find.rbi
182
+ - rbi/generated/prism/parse_result.rbi
183
+ - rbi/generated/prism/parse_result/errors.rbi
184
+ - rbi/generated/prism/parse_result/newlines.rbi
185
+ - rbi/generated/prism/pattern.rbi
186
+ - rbi/generated/prism/reflection.rbi
187
+ - rbi/generated/prism/relocation.rbi
188
+ - rbi/generated/prism/serialize.rbi
189
+ - rbi/generated/prism/string_query.rbi
190
+ - rbi/generated/prism/translation.rbi
191
+ - rbi/generated/prism/visitor.rbi
125
192
  - rbi/prism/translation/parser.rbi
126
193
  - rbi/prism/translation/parser_versions.rbi
127
194
  - rbi/prism/translation/ripper.rbi
128
- - rbi/prism/visitor.rbi
129
- - sig/prism.rbs
130
- - sig/prism/compiler.rbs
131
- - sig/prism/dispatcher.rbs
132
- - sig/prism/dot_visitor.rbs
133
- - sig/prism/dsl.rbs
134
- - sig/prism/inspect_visitor.rbs
135
- - sig/prism/lex_compat.rbs
136
- - sig/prism/mutation_compiler.rbs
137
- - sig/prism/node.rbs
138
- - sig/prism/node_ext.rbs
139
- - sig/prism/pack.rbs
140
- - sig/prism/parse_result.rbs
141
- - sig/prism/parse_result/comments.rbs
142
- - sig/prism/pattern.rbs
143
- - sig/prism/reflection.rbs
144
- - sig/prism/relocation.rbs
145
- - sig/prism/serialize.rbs
146
- - sig/prism/string_query.rbs
147
- - sig/prism/visitor.rbs
195
+ - rbi/rubyvm/node_find.rbi
196
+ - sig/generated/prism.rbs
197
+ - sig/generated/prism/compiler.rbs
198
+ - sig/generated/prism/desugar_compiler.rbs
199
+ - sig/generated/prism/dispatcher.rbs
200
+ - sig/generated/prism/dot_visitor.rbs
201
+ - sig/generated/prism/dsl.rbs
202
+ - sig/generated/prism/inspect_visitor.rbs
203
+ - sig/generated/prism/lex_compat.rbs
204
+ - sig/generated/prism/mutation_compiler.rbs
205
+ - sig/generated/prism/node.rbs
206
+ - sig/generated/prism/node_ext.rbs
207
+ - sig/generated/prism/node_find.rbs
208
+ - sig/generated/prism/parse_result.rbs
209
+ - sig/generated/prism/parse_result/errors.rbs
210
+ - sig/generated/prism/parse_result/newlines.rbs
211
+ - sig/generated/prism/pattern.rbs
212
+ - sig/generated/prism/reflection.rbs
213
+ - sig/generated/prism/relocation.rbs
214
+ - sig/generated/prism/serialize.rbs
215
+ - sig/generated/prism/string_query.rbs
216
+ - sig/generated/prism/translation.rbs
217
+ - sig/generated/prism/visitor.rbs
218
+ - sig/manifest.yaml
219
+ - src/arena.c
220
+ - src/buffer.c
221
+ - src/char.c
222
+ - src/comments.c
223
+ - src/constant_pool.c
148
224
  - src/diagnostic.c
149
225
  - src/encoding.c
226
+ - src/errors_format.c
227
+ - src/integer.c
228
+ - src/json.c
229
+ - src/line_offset_list.c
230
+ - src/list.c
231
+ - src/memchr.c
150
232
  - src/node.c
151
233
  - src/options.c
152
- - src/pack.c
234
+ - src/parser.c
153
235
  - src/prettyprint.c
154
236
  - src/prism.c
155
237
  - src/regexp.c
156
238
  - src/serialize.c
239
+ - src/source.c
157
240
  - src/static_literals.c
158
- - src/token_type.c
159
- - src/util/pm_buffer.c
160
- - src/util/pm_char.c
161
- - src/util/pm_constant_pool.c
162
- - src/util/pm_integer.c
163
- - src/util/pm_list.c
164
- - src/util/pm_memchr.c
165
- - src/util/pm_newline_list.c
166
- - src/util/pm_string.c
167
- - src/util/pm_strncasecmp.c
168
- - src/util/pm_strpbrk.c
241
+ - src/string_query.c
242
+ - src/stringy.c
243
+ - src/strncasecmp.c
244
+ - src/strpbrk.c
245
+ - src/tokens.c
246
+ - src/xxh3.c
169
247
  homepage: https://github.com/ruby/prism
170
248
  licenses:
171
249
  - MIT
@@ -187,7 +265,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
187
265
  - !ruby/object:Gem::Version
188
266
  version: '0'
189
267
  requirements: []
190
- rubygems_version: 3.6.9
268
+ rubygems_version: 4.0.20
191
269
  specification_version: 4
192
270
  summary: Prism Ruby parser
193
271
  test_files: []