prism 1.9.0 → 2.0.0.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +41 -12
  6. data/README.md +35 -97
  7. data/config.yml +174 -161
  8. data/docs/build_system.md +2 -2
  9. data/docs/configuration.md +5 -4
  10. data/docs/encoding.md +2 -2
  11. data/docs/fuzzing.md +4 -11
  12. data/docs/javascript.md +19 -4
  13. data/docs/mapping.md +1 -1
  14. data/docs/releasing.md +77 -5
  15. data/docs/relocation.md +2 -4
  16. data/docs/serialization.md +22 -24
  17. data/docs/testing.md +46 -29
  18. data/ext/prism/api_node.c +1461 -1401
  19. data/ext/prism/extconf.rb +3 -4
  20. data/ext/prism/extension.c +796 -422
  21. data/ext/prism/extension.h +3 -3
  22. data/include/prism/arena.h +37 -0
  23. data/include/prism/ast.h +2727 -610
  24. data/include/prism/buffer.h +52 -0
  25. data/include/prism/comments.h +55 -0
  26. data/include/prism/compiler/accel.h +26 -0
  27. data/include/prism/compiler/align.h +36 -0
  28. data/include/prism/compiler/assume.h +25 -0
  29. data/include/prism/compiler/bswap.h +46 -0
  30. data/include/prism/compiler/endianness.h +34 -0
  31. data/include/prism/compiler/exported.h +24 -0
  32. data/include/prism/compiler/fallthrough.h +22 -0
  33. data/include/prism/compiler/filesystem.h +32 -0
  34. data/include/prism/compiler/flex_array.h +19 -0
  35. data/include/prism/compiler/force_inline.h +21 -0
  36. data/include/prism/compiler/format.h +25 -0
  37. data/include/prism/compiler/inline.h +17 -0
  38. data/include/prism/compiler/nodiscard.h +22 -0
  39. data/include/prism/compiler/nonnull.h +18 -0
  40. data/include/prism/compiler/unused.h +18 -0
  41. data/include/prism/constant_pool.h +81 -0
  42. data/include/prism/diagnostic.h +38 -403
  43. data/include/prism/errors_format.h +53 -0
  44. data/include/prism/excludes.h +29 -0
  45. data/include/prism/integer.h +41 -0
  46. data/include/prism/internal/allocator.h +68 -0
  47. data/include/prism/internal/allocator_debug.h +88 -0
  48. data/include/prism/internal/arena.h +108 -0
  49. data/include/prism/internal/bit.h +42 -0
  50. data/include/prism/internal/buffer.h +94 -0
  51. data/include/prism/internal/char.h +145 -0
  52. data/include/prism/internal/comments.h +53 -0
  53. data/include/prism/internal/constant_pool.h +152 -0
  54. data/include/prism/internal/diagnostic.h +389 -0
  55. data/include/prism/{encoding.h → internal/encoding.h} +27 -68
  56. data/include/prism/internal/integer.h +68 -0
  57. data/include/prism/internal/isinf.h +16 -0
  58. data/include/prism/internal/line_offset_list.h +34 -0
  59. data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  60. data/include/prism/internal/magic_comments.h +23 -0
  61. data/include/prism/internal/memchr.h +15 -0
  62. data/include/prism/internal/node.h +38 -0
  63. data/include/prism/internal/options.h +232 -0
  64. data/include/prism/internal/parser.h +1004 -0
  65. data/include/prism/internal/regexp.h +26 -0
  66. data/include/prism/internal/serialize.h +39 -0
  67. data/include/prism/internal/source.h +72 -0
  68. data/include/prism/{static_literals.h → internal/static_literals.h} +26 -47
  69. data/include/prism/internal/stringy.h +35 -0
  70. data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  71. data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  72. data/include/prism/internal/tokens.h +11 -0
  73. data/include/prism/internal/xxh3.h +16 -0
  74. data/include/prism/json.h +32 -0
  75. data/include/prism/line_offset_list.h +61 -0
  76. data/include/prism/magic_comments.h +35 -0
  77. data/include/prism/node.h +18 -53
  78. data/include/prism/options.h +146 -280
  79. data/include/prism/parser.h +356 -867
  80. data/include/prism/prettyprint.h +6 -9
  81. data/include/prism/serialize.h +115 -0
  82. data/include/prism/source.h +155 -0
  83. data/include/prism/stream.h +28 -0
  84. data/include/prism/string_query.h +74 -0
  85. data/include/prism/stringy.h +72 -0
  86. data/include/prism/version.h +12 -3
  87. data/include/prism.h +43 -320
  88. data/lib/prism/compiler.rb +319 -305
  89. data/lib/prism/desugar_compiler.rb +105 -34
  90. data/lib/prism/dispatcher.rb +646 -473
  91. data/lib/prism/dot_visitor.rb +592 -511
  92. data/lib/prism/dsl.rb +391 -25
  93. data/lib/prism/ffi.rb +258 -90
  94. data/lib/prism/inspect_visitor.rb +357 -325
  95. data/lib/prism/lex_compat.rb +254 -212
  96. data/lib/prism/mutation_compiler.rb +313 -305
  97. data/lib/prism/node.rb +20966 -6992
  98. data/lib/prism/node_ext.rb +180 -187
  99. data/lib/prism/node_find.rb +215 -0
  100. data/lib/prism/parse_result/errors.rb +7 -1
  101. data/lib/prism/parse_result/newlines.rb +276 -8
  102. data/lib/prism/parse_result.rb +468 -139
  103. data/lib/prism/pattern.rb +65 -20
  104. data/lib/prism/reflection.rb +23 -12
  105. data/lib/prism/relocation.rb +149 -87
  106. data/lib/prism/serialize.rb +3460 -539
  107. data/lib/prism/string_query.rb +24 -1
  108. data/lib/prism/translation/parser/builder.rb +10 -2
  109. data/lib/prism/translation/parser/compiler.rb +44 -52
  110. data/lib/prism/translation/parser/lexer.rb +41 -92
  111. data/lib/prism/translation/parser.rb +1 -1
  112. data/lib/prism/translation/parser_current.rb +1 -1
  113. data/lib/prism/translation/ripper/lexer.rb +8 -10
  114. data/lib/prism/translation/ripper/sexp.rb +2 -10
  115. data/lib/prism/translation/ripper/shim.rb +2 -0
  116. data/lib/prism/translation/ripper.rb +967 -209
  117. data/lib/prism/translation/ruby_parser.rb +73 -316
  118. data/lib/prism/translation.rb +2 -0
  119. data/lib/prism/visitor.rb +632 -5
  120. data/lib/prism.rb +79 -9
  121. data/prism.gemspec +128 -62
  122. data/rbi/generated/prism/compiler.rbi +489 -0
  123. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  124. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  125. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  126. data/rbi/generated/prism/dsl.rbi +751 -0
  127. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  128. data/rbi/generated/prism/lex_compat.rbi +162 -0
  129. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  130. data/rbi/generated/prism/node.rbi +14407 -0
  131. data/rbi/generated/prism/node_ext.rbi +259 -0
  132. data/rbi/generated/prism/node_find.rbi +79 -0
  133. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  134. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  135. data/rbi/generated/prism/parse_result.rbi +776 -0
  136. data/rbi/generated/prism/pattern.rbi +134 -0
  137. data/rbi/generated/prism/reflection.rbi +89 -0
  138. data/rbi/generated/prism/relocation.rbi +329 -0
  139. data/rbi/generated/prism/serialize.rbi +184 -0
  140. data/rbi/generated/prism/string_query.rbi +43 -0
  141. data/rbi/generated/prism/translation.rbi +7 -0
  142. data/rbi/generated/prism/visitor.rbi +650 -0
  143. data/rbi/generated/prism.rbi +86 -0
  144. data/rbi/prism/translation/parser_versions.rbi +5 -0
  145. data/rbi/rubyvm/node_find.rbi +14 -0
  146. data/sig/generated/prism/compiler.rbs +492 -0
  147. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  148. data/sig/generated/prism/dispatcher.rbs +1292 -0
  149. data/sig/generated/prism/dot_visitor.rbs +635 -0
  150. data/sig/generated/prism/dsl.rbs +928 -0
  151. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  152. data/sig/generated/prism/lex_compat.rbs +166 -0
  153. data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
  154. data/sig/generated/prism/node.rbs +20896 -0
  155. data/sig/generated/prism/node_ext.rbs +290 -0
  156. data/sig/generated/prism/node_find.rbs +90 -0
  157. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  158. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  159. data/sig/generated/prism/parse_result.rbs +871 -0
  160. data/sig/generated/prism/pattern.rbs +156 -0
  161. data/sig/generated/prism/reflection.rbs +101 -0
  162. data/sig/generated/prism/relocation.rbs +428 -0
  163. data/sig/generated/prism/serialize.rbs +198 -0
  164. data/sig/generated/prism/string_query.rbs +43 -0
  165. data/sig/generated/prism/translation.rbs +8 -0
  166. data/sig/generated/prism/visitor.rbs +1111 -0
  167. data/sig/generated/prism.rbs +84 -0
  168. data/sig/manifest.yaml +2 -0
  169. data/src/arena.c +117 -0
  170. data/src/{util/pm_buffer.c → buffer.c} +53 -21
  171. data/src/{util/pm_char.c → char.c} +28 -57
  172. data/src/comments.c +454 -0
  173. data/src/constant_pool.c +406 -0
  174. data/src/diagnostic.c +97 -65
  175. data/src/encoding.c +8 -3
  176. data/src/errors_format.c +519 -0
  177. data/src/{util/pm_integer.c → integer.c} +33 -22
  178. data/src/json.c +5733 -0
  179. data/src/line_offset_list.c +100 -0
  180. data/src/list.c +24 -0
  181. data/src/{util/pm_memchr.c → memchr.c} +8 -6
  182. data/src/node.c +3504 -7674
  183. data/src/options.c +162 -49
  184. data/src/parser.c +367 -0
  185. data/src/prettyprint.c +421 -354
  186. data/src/prism.c +8402 -7161
  187. data/src/regexp.c +961 -45
  188. data/src/serialize.c +460 -424
  189. data/src/source.c +518 -0
  190. data/src/static_literals.c +111 -43
  191. data/src/string_query.c +180 -0
  192. data/src/stringy.c +91 -0
  193. data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  194. data/src/strpbrk.c +447 -0
  195. data/src/{token_type.c → tokens.c} +39 -18
  196. data/src/xxh3.c +292 -0
  197. metadata +128 -63
  198. data/ext/prism/api_pack.c +0 -276
  199. data/include/prism/defines.h +0 -260
  200. data/include/prism/pack.h +0 -163
  201. data/include/prism/regexp.h +0 -43
  202. data/include/prism/util/pm_buffer.h +0 -236
  203. data/include/prism/util/pm_char.h +0 -204
  204. data/include/prism/util/pm_constant_pool.h +0 -218
  205. data/include/prism/util/pm_integer.h +0 -130
  206. data/include/prism/util/pm_memchr.h +0 -29
  207. data/include/prism/util/pm_newline_list.h +0 -113
  208. data/include/prism/util/pm_string.h +0 -200
  209. data/lib/prism/pack.rb +0 -230
  210. data/lib/prism/parse_result/comments.rb +0 -188
  211. data/rbi/prism/compiler.rbi +0 -12
  212. data/rbi/prism/dsl.rbi +0 -524
  213. data/rbi/prism/inspect_visitor.rbi +0 -12
  214. data/rbi/prism/node.rbi +0 -8750
  215. data/rbi/prism/node_ext.rbi +0 -107
  216. data/rbi/prism/parse_result.rbi +0 -404
  217. data/rbi/prism/reflection.rbi +0 -58
  218. data/rbi/prism/string_query.rbi +0 -12
  219. data/rbi/prism/visitor.rbi +0 -473
  220. data/rbi/prism.rbi +0 -63
  221. data/sig/prism/compiler.rbs +0 -9
  222. data/sig/prism/dispatcher.rbs +0 -19
  223. data/sig/prism/dot_visitor.rbs +0 -6
  224. data/sig/prism/dsl.rbs +0 -351
  225. data/sig/prism/inspect_visitor.rbs +0 -22
  226. data/sig/prism/lex_compat.rbs +0 -10
  227. data/sig/prism/node.rbs +0 -4042
  228. data/sig/prism/node_ext.rbs +0 -149
  229. data/sig/prism/pack.rbs +0 -43
  230. data/sig/prism/parse_result/comments.rbs +0 -38
  231. data/sig/prism/parse_result.rbs +0 -197
  232. data/sig/prism/pattern.rbs +0 -13
  233. data/sig/prism/reflection.rbs +0 -50
  234. data/sig/prism/relocation.rbs +0 -185
  235. data/sig/prism/serialize.rbs +0 -8
  236. data/sig/prism/string_query.rbs +0 -11
  237. data/sig/prism/visitor.rbs +0 -169
  238. data/sig/prism.rbs +0 -272
  239. data/src/pack.c +0 -509
  240. data/src/util/pm_constant_pool.c +0 -342
  241. data/src/util/pm_list.c +0 -49
  242. data/src/util/pm_newline_list.c +0 -125
  243. data/src/util/pm_string.c +0 -381
  244. data/src/util/pm_strpbrk.c +0 -206
data/src/xxh3.c ADDED
@@ -0,0 +1,292 @@
1
+ /**
2
+ * An implementation of the XXH3 hash function.
3
+ *
4
+ * The reason some of these names do not look quite like the style of the rest
5
+ * of the code in Prism is that this was adapted straight from the XXH3
6
+ * specification here:
7
+ *
8
+ * https://github.com/Cyan4973/xxHash/blob/release/doc/xxhash_spec.md
9
+ *
10
+ * To better understand this code it is recommended to read the XXH3
11
+ * specification linked above, where you should be able to follow along with
12
+ * this implementation more easily. Note this only implements the XXH3_64
13
+ * variant with the default secret and a seed of 0. Additionally it does not
14
+ * include a lot of the various CPU-specific optimizations that exist in the
15
+ * reference implementation.
16
+ */
17
+
18
+ #include "prism/internal/xxh3.h"
19
+
20
+ #include "prism/compiler/accel.h"
21
+ #include "prism/compiler/bswap.h"
22
+ #include "prism/compiler/endianness.h"
23
+
24
+ #include <string.h>
25
+
26
+ #if defined(PRISM_HAS_NEON) && defined(PRISM_LITTLE_ENDIAN_COMPTIME)
27
+ #include <arm_neon.h>
28
+ #define XXH3_NEON
29
+ #elif defined(PRISM_HAS_SSE2)
30
+ #include <emmintrin.h>
31
+ #define XXH3_SSE2
32
+ #endif
33
+
34
+ #if defined(__SIZEOF_INT128__)
35
+ static uint64_t
36
+ XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
37
+ __uint128_t x = (__uint128_t) lhs * (__uint128_t) rhs;
38
+ return ((uint64_t) (x & 0xFFFFFFFFFFFFFFFFULL)) ^ ((uint64_t) (x >> 64));
39
+ }
40
+ #else
41
+ #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64))
42
+ #include <intrin.h>
43
+ #endif
44
+
45
+ static uint64_t
46
+ XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
47
+ uint64_t lo, hi;
48
+ #if defined(_MSC_VER) && defined(_M_X64)
49
+ lo = _umul128(lhs, rhs, &hi);
50
+ #elif defined(_MSC_VER) && defined(_M_ARM64)
51
+ hi = __umulh(lhs, rhs);
52
+ lo = lhs * rhs;
53
+ #else
54
+ uint64_t lo_lo = (lhs & 0xFFFFFFFF) * (rhs & 0xFFFFFFFF);
55
+ uint64_t hi_lo = (lhs >> 32) * (rhs & 0xFFFFFFFF);
56
+ uint64_t lo_hi = (lhs & 0xFFFFFFFF) * (rhs >> 32);
57
+ uint64_t hi_hi = (lhs >> 32) * (rhs >> 32);
58
+
59
+ uint64_t cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
60
+ hi = hi_hi + (hi_lo >> 32) + (cross >> 32);
61
+ lo = (cross << 32) | (lo_lo & 0xFFFFFFFF);
62
+ #endif
63
+ return lo ^ hi;
64
+ }
65
+ #endif
66
+
67
+ static uint32_t
68
+ read32(const void* ptr) {
69
+ uint32_t val;
70
+ memcpy(&val, ptr, sizeof(val));
71
+ if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
72
+ val = pm_bswap32(val);
73
+ }
74
+ return val;
75
+ }
76
+
77
+ static uint64_t
78
+ read64(const void* ptr) {
79
+ uint64_t val;
80
+ memcpy(&val, ptr, sizeof(val));
81
+ if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
82
+ val = pm_bswap64(val);
83
+ }
84
+ return val;
85
+ }
86
+
87
+ static const uint64_t PRIME32_1 = 0x9E3779B1U;
88
+ static const uint64_t PRIME32_2 = 0x85EBCA77U;
89
+ static const uint64_t PRIME32_3 = 0xC2B2AE3DU;
90
+ static const uint64_t PRIME64_1 = 0x9E3779B185EBCA87ULL;
91
+ static const uint64_t PRIME64_2 = 0xC2B2AE3D27D4EB4FULL;
92
+ static const uint64_t PRIME64_3 = 0x165667B19E3779F9ULL;
93
+ static const uint64_t PRIME64_4 = 0x85EBCA77C2B2AE63ULL;
94
+ static const uint64_t PRIME64_5 = 0x27D4EB2F165667C5ULL;
95
+ static const uint64_t PRIME_MX1 = 0x165667919E3779F9ULL;
96
+ static const uint64_t PRIME_MX2 = 0x9FB21C651E98DF25ULL;
97
+
98
+ static const uint8_t secret[192] = {
99
+ 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
100
+ 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
101
+ 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
102
+ 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
103
+ 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
104
+ 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
105
+ 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
106
+ 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
107
+ 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
108
+ 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
109
+ 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
110
+ 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
111
+ };
112
+
113
+ static uint64_t
114
+ avalanche(uint64_t x) {
115
+ x ^= (x >> 37);
116
+ x *= PRIME_MX1;
117
+ x ^= (x >> 32);
118
+ return x;
119
+ }
120
+
121
+ static uint64_t
122
+ avalanche_XXH64(uint64_t x) {
123
+ x ^= (x >> 33);
124
+ x *= PRIME64_2;
125
+ x ^= (x >> 29);
126
+ x *= PRIME64_3;
127
+ x ^= (x >> 32);
128
+ return x;
129
+ }
130
+
131
+ static uint64_t
132
+ XXH3_64_empty(void) {
133
+ return avalanche_XXH64(read64(secret + 56) ^ read64(secret + 64));
134
+ }
135
+
136
+ static uint64_t
137
+ XXH3_64_1to3(const uint8_t *input, size_t length) {
138
+ uint32_t combined = (uint32_t) input[length - 1] | ((uint32_t) length << 8) | ((uint32_t) input[0] << 16) | ((uint32_t) input[length >> 1] << 24);
139
+ return avalanche_XXH64((uint64_t) (read32(secret) ^ read32(secret + 4)) ^ (uint64_t) combined);
140
+ }
141
+
142
+ static uint64_t
143
+ XXH3_64_4to8(const uint8_t *input, size_t length) {
144
+ #define rot(x_, r_) (((x_) << (r_)) | ((x_) >> (64 - (r_))))
145
+
146
+ uint64_t combined = (uint64_t) read32(input + length - 4) | ((uint64_t) read32(input) << 32);
147
+ uint64_t value = (read64(secret + 8) ^ read64(secret + 16)) ^ combined;
148
+ value ^= rot(value, 49) ^ rot(value, 24);
149
+ value *= PRIME_MX2;
150
+ value ^= ((value >> 35) + length);
151
+ value *= PRIME_MX2;
152
+ value ^= (value >> 28);
153
+ return value;
154
+
155
+ #undef rot
156
+ }
157
+
158
+ static uint64_t
159
+ XXH3_64_9to16(const uint8_t *input, size_t length) {
160
+ uint64_t lo = (read64(secret + 24) ^ read64(secret + 32)) ^ read64(input);
161
+ uint64_t hi = (read64(secret + 40) ^ read64(secret + 48)) ^ read64(input + length - 8);
162
+ return avalanche(length + pm_bswap64(lo) + hi + XXH3_mul128_fold64(lo, hi));
163
+ }
164
+
165
+ static uint64_t
166
+ XXH3_64_17to240(const uint8_t *input, size_t length) {
167
+ #define mix(data_, offset_) XXH3_mul128_fold64(read64((data_)) ^ read64(secret + (offset_)), read64((data_) + 8) ^ read64(secret + (offset_) + 8))
168
+
169
+ uint64_t acc = length * PRIME64_1;
170
+
171
+ if (length <= 128) {
172
+ size_t idx = (length - 1) / 32;
173
+ do {
174
+ acc += mix(input + idx * 16, idx * 32) + mix(input + length - idx * 16 - 16, idx * 32 + 16);
175
+ } while (idx--);
176
+ } else {
177
+ for (size_t idx = 0; idx < 8; idx++) {
178
+ acc += mix(input + idx * 16, idx * 16);
179
+ }
180
+ acc = avalanche(acc);
181
+ for (size_t idx = 8; idx < length >> 4; idx++) {
182
+ acc += mix(input + idx * 16, (idx - 8) * 16 + 3);
183
+ }
184
+ acc += mix(input + length - 16, 119);
185
+ }
186
+
187
+ return avalanche(acc);
188
+
189
+ #undef mix
190
+ }
191
+
192
+ static void
193
+ accumulate(uint64_t acc[8], const uint8_t *stripe, size_t secret_offset) {
194
+ #if defined(XXH3_NEON)
195
+ for (int idx = 0; idx < 8; idx += 2) {
196
+ uint64x2_t data = vreinterpretq_u64_u8(vld1q_u8(stripe + 8 * idx));
197
+ uint64x2_t value = veorq_u64(data, vreinterpretq_u64_u8(vld1q_u8(secret + secret_offset + 8 * idx)));
198
+ uint64x2_t product = vmull_u32(vmovn_u64(value), vshrn_n_u64(value, 32));
199
+ vst1q_u64(acc + idx, vaddq_u64(vld1q_u64(acc + idx), vaddq_u64(vextq_u64(data, data, 1), product)));
200
+ }
201
+ #elif defined(XXH3_SSE2)
202
+ for (int idx = 0; idx < 8; idx += 2) {
203
+ __m128i data = _mm_loadu_si128((const __m128i *) (stripe + 8 * idx));
204
+ __m128i value = _mm_xor_si128(data, _mm_loadu_si128((const __m128i *) (secret + secret_offset + 8 * idx)));
205
+ __m128i product = _mm_mul_epu32(value, _mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)));
206
+ _mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_loadu_si128((const __m128i *) (acc + idx)), _mm_add_epi64(_mm_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2)), product)));
207
+ }
208
+ #else
209
+ for (int idx = 0; idx < 8; idx++) {
210
+ uint64_t data = read64(stripe + 8 * idx);
211
+ uint64_t value = data ^ read64(secret + secret_offset + 8 * idx);
212
+ acc[idx ^ 1] = acc[idx ^ 1] + data;
213
+ acc[idx] += (value & 0xFFFFFFFF) * (value >> 32);
214
+ }
215
+ #endif
216
+ }
217
+
218
+ static uint64_t
219
+ XXH3_64_large(const uint8_t *input, size_t length) {
220
+ const size_t secret_length = sizeof(secret);
221
+ const size_t stripes_per_block = (secret_length - 64) / 8;
222
+ const size_t block_size = 64 * stripes_per_block;
223
+ const size_t block_count = (length - 1) / block_size;
224
+ const size_t last_offset = block_count * block_size;
225
+
226
+ uint64_t acc[8] = {
227
+ PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3,
228
+ PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1
229
+ };
230
+
231
+ for (size_t block_idx = 0; block_idx < block_count; block_idx++) {
232
+ const uint8_t *block = input + block_idx * block_size;
233
+
234
+ for (size_t stripe_idx = 0; stripe_idx < stripes_per_block; stripe_idx++) {
235
+ accumulate(acc, block + stripe_idx * 64, stripe_idx * 8);
236
+ }
237
+
238
+ #if defined(XXH3_NEON)
239
+ const uint32x2_t prime = vdup_n_u32((uint32_t) PRIME32_1);
240
+ for (size_t idx = 0; idx < 8; idx += 2) {
241
+ uint64x2_t value = vld1q_u64(acc + idx);
242
+ value = veorq_u64(value, vshrq_n_u64(value, 47));
243
+ value = veorq_u64(value, vreinterpretq_u64_u8(vld1q_u8(secret + secret_length - 64 + idx * 8)));
244
+ vst1q_u64(acc + idx, vaddq_u64(vmull_u32(vmovn_u64(value), prime), vshlq_n_u64(vmull_u32(vshrn_n_u64(value, 32), prime), 32)));
245
+ }
246
+ #elif defined(XXH3_SSE2)
247
+ const __m128i prime = _mm_set1_epi32((int) (uint32_t) PRIME32_1);
248
+ for (size_t idx = 0; idx < 8; idx += 2) {
249
+ __m128i value = _mm_loadu_si128((const __m128i *) (acc + idx));
250
+ value = _mm_xor_si128(value, _mm_srli_epi64(value, 47));
251
+ value = _mm_xor_si128(value, _mm_loadu_si128((const __m128i *) (secret + secret_length - 64 + idx * 8)));
252
+ _mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_mul_epu32(value, prime), _mm_slli_epi64(_mm_mul_epu32(_mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)), prime), 32)));
253
+ }
254
+ #else
255
+ for (size_t idx = 0; idx < 8; idx++) {
256
+ acc[idx] ^= (acc[idx] >> 47);
257
+ acc[idx] ^= read64(secret + secret_length - 64 + idx * 8);
258
+ acc[idx] *= PRIME32_1;
259
+ }
260
+ #endif
261
+ }
262
+
263
+ for (size_t stripe_idx = 0; stripe_idx < (length - last_offset - 1) / 64; stripe_idx++) {
264
+ accumulate(acc, input + last_offset + stripe_idx * 64, stripe_idx * 8);
265
+ }
266
+
267
+ accumulate(acc, input + length - 64, secret_length - 71);
268
+
269
+ uint64_t result = (uint64_t) length * PRIME64_1;
270
+ for (size_t idx = 0; idx < 4; idx++) {
271
+ result += XXH3_mul128_fold64(acc[idx * 2] ^ read64(secret + 11 + 16 * idx), acc[idx * 2 + 1] ^ read64(secret + 11 + 16 * idx + 8));
272
+ }
273
+
274
+ return avalanche(result);
275
+ }
276
+
277
+ uint64_t
278
+ pm_xxh3(const uint8_t *input, size_t length) {
279
+ if (length == 0) {
280
+ return XXH3_64_empty();
281
+ } else if (length <= 3) {
282
+ return XXH3_64_1to3(input, length);
283
+ } else if (length <= 8) {
284
+ return XXH3_64_4to8(input, length);
285
+ } else if (length <= 16) {
286
+ return XXH3_64_9to16(input, length);
287
+ } else if (length <= 240) {
288
+ return XXH3_64_17to240(input, length);
289
+ } else {
290
+ return XXH3_64_large(input, length);
291
+ }
292
+ }
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: prism
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.9.0
4
+ version: 2.0.0.rc2
5
5
  platform: ruby
6
6
  authors:
7
- - Shopify
7
+ - Ruby developers
8
8
  bindir: bin
9
9
  cert_chain: []
10
10
  date: 1980-01-02 00:00:00.000000000 Z
11
11
  dependencies: []
12
12
  email:
13
- - ruby@shopify.com
13
+ - ruby-core@ruby-lang.org
14
14
  executables: []
15
15
  extensions:
16
16
  - ext/prism/extconf.rb
@@ -44,32 +44,74 @@ files:
44
44
  - docs/serialization.md
45
45
  - docs/testing.md
46
46
  - ext/prism/api_node.c
47
- - ext/prism/api_pack.c
48
47
  - ext/prism/extconf.rb
49
48
  - ext/prism/extension.c
50
49
  - ext/prism/extension.h
51
50
  - include/prism.h
51
+ - include/prism/arena.h
52
52
  - include/prism/ast.h
53
- - include/prism/defines.h
53
+ - include/prism/buffer.h
54
+ - include/prism/comments.h
55
+ - include/prism/compiler/accel.h
56
+ - include/prism/compiler/align.h
57
+ - include/prism/compiler/assume.h
58
+ - include/prism/compiler/bswap.h
59
+ - include/prism/compiler/endianness.h
60
+ - include/prism/compiler/exported.h
61
+ - include/prism/compiler/fallthrough.h
62
+ - include/prism/compiler/filesystem.h
63
+ - include/prism/compiler/flex_array.h
64
+ - include/prism/compiler/force_inline.h
65
+ - include/prism/compiler/format.h
66
+ - include/prism/compiler/inline.h
67
+ - include/prism/compiler/nodiscard.h
68
+ - include/prism/compiler/nonnull.h
69
+ - include/prism/compiler/unused.h
70
+ - include/prism/constant_pool.h
54
71
  - include/prism/diagnostic.h
55
- - include/prism/encoding.h
72
+ - include/prism/errors_format.h
73
+ - include/prism/excludes.h
74
+ - include/prism/integer.h
75
+ - include/prism/internal/allocator.h
76
+ - include/prism/internal/allocator_debug.h
77
+ - include/prism/internal/arena.h
78
+ - include/prism/internal/bit.h
79
+ - include/prism/internal/buffer.h
80
+ - include/prism/internal/char.h
81
+ - include/prism/internal/comments.h
82
+ - include/prism/internal/constant_pool.h
83
+ - include/prism/internal/diagnostic.h
84
+ - include/prism/internal/encoding.h
85
+ - include/prism/internal/integer.h
86
+ - include/prism/internal/isinf.h
87
+ - include/prism/internal/line_offset_list.h
88
+ - include/prism/internal/list.h
89
+ - include/prism/internal/magic_comments.h
90
+ - include/prism/internal/memchr.h
91
+ - include/prism/internal/node.h
92
+ - include/prism/internal/options.h
93
+ - include/prism/internal/parser.h
94
+ - include/prism/internal/regexp.h
95
+ - include/prism/internal/serialize.h
96
+ - include/prism/internal/source.h
97
+ - include/prism/internal/static_literals.h
98
+ - include/prism/internal/stringy.h
99
+ - include/prism/internal/strncasecmp.h
100
+ - include/prism/internal/strpbrk.h
101
+ - include/prism/internal/tokens.h
102
+ - include/prism/internal/xxh3.h
103
+ - include/prism/json.h
104
+ - include/prism/line_offset_list.h
105
+ - include/prism/magic_comments.h
56
106
  - include/prism/node.h
57
107
  - include/prism/options.h
58
- - include/prism/pack.h
59
108
  - include/prism/parser.h
60
109
  - include/prism/prettyprint.h
61
- - include/prism/regexp.h
62
- - include/prism/static_literals.h
63
- - include/prism/util/pm_buffer.h
64
- - include/prism/util/pm_char.h
65
- - include/prism/util/pm_constant_pool.h
66
- - include/prism/util/pm_integer.h
67
- - include/prism/util/pm_list.h
68
- - include/prism/util/pm_memchr.h
69
- - include/prism/util/pm_newline_list.h
70
- - include/prism/util/pm_string.h
71
- - include/prism/util/pm_strncasecmp.h
72
- - include/prism/util/pm_strpbrk.h
110
+ - include/prism/serialize.h
111
+ - include/prism/source.h
112
+ - include/prism/stream.h
113
+ - include/prism/string_query.h
114
+ - include/prism/stringy.h
73
115
  - include/prism/version.h
74
116
  - lib/prism.rb
75
117
  - lib/prism/compiler.rb
@@ -83,9 +125,8 @@ files:
83
125
  - lib/prism/mutation_compiler.rb
84
126
  - lib/prism/node.rb
85
127
  - lib/prism/node_ext.rb
86
- - lib/prism/pack.rb
128
+ - lib/prism/node_find.rb
87
129
  - lib/prism/parse_result.rb
88
- - lib/prism/parse_result/comments.rb
89
130
  - lib/prism/parse_result/errors.rb
90
131
  - lib/prism/parse_result/newlines.rb
91
132
  - lib/prism/pattern.rb
@@ -113,59 +154,83 @@ files:
113
154
  - lib/prism/translation/ruby_parser.rb
114
155
  - lib/prism/visitor.rb
115
156
  - prism.gemspec
116
- - rbi/prism.rbi
117
- - rbi/prism/compiler.rbi
118
- - rbi/prism/dsl.rbi
119
- - rbi/prism/inspect_visitor.rbi
120
- - rbi/prism/node.rbi
121
- - rbi/prism/node_ext.rbi
122
- - rbi/prism/parse_result.rbi
123
- - rbi/prism/reflection.rbi
124
- - rbi/prism/string_query.rbi
157
+ - rbi/generated/prism.rbi
158
+ - rbi/generated/prism/compiler.rbi
159
+ - rbi/generated/prism/desugar_compiler.rbi
160
+ - rbi/generated/prism/dispatcher.rbi
161
+ - rbi/generated/prism/dot_visitor.rbi
162
+ - rbi/generated/prism/dsl.rbi
163
+ - rbi/generated/prism/inspect_visitor.rbi
164
+ - rbi/generated/prism/lex_compat.rbi
165
+ - rbi/generated/prism/mutation_compiler.rbi
166
+ - rbi/generated/prism/node.rbi
167
+ - rbi/generated/prism/node_ext.rbi
168
+ - rbi/generated/prism/node_find.rbi
169
+ - rbi/generated/prism/parse_result.rbi
170
+ - rbi/generated/prism/parse_result/errors.rbi
171
+ - rbi/generated/prism/parse_result/newlines.rbi
172
+ - rbi/generated/prism/pattern.rbi
173
+ - rbi/generated/prism/reflection.rbi
174
+ - rbi/generated/prism/relocation.rbi
175
+ - rbi/generated/prism/serialize.rbi
176
+ - rbi/generated/prism/string_query.rbi
177
+ - rbi/generated/prism/translation.rbi
178
+ - rbi/generated/prism/visitor.rbi
125
179
  - rbi/prism/translation/parser.rbi
126
180
  - rbi/prism/translation/parser_versions.rbi
127
181
  - rbi/prism/translation/ripper.rbi
128
- - rbi/prism/visitor.rbi
129
- - sig/prism.rbs
130
- - sig/prism/compiler.rbs
131
- - sig/prism/dispatcher.rbs
132
- - sig/prism/dot_visitor.rbs
133
- - sig/prism/dsl.rbs
134
- - sig/prism/inspect_visitor.rbs
135
- - sig/prism/lex_compat.rbs
136
- - sig/prism/mutation_compiler.rbs
137
- - sig/prism/node.rbs
138
- - sig/prism/node_ext.rbs
139
- - sig/prism/pack.rbs
140
- - sig/prism/parse_result.rbs
141
- - sig/prism/parse_result/comments.rbs
142
- - sig/prism/pattern.rbs
143
- - sig/prism/reflection.rbs
144
- - sig/prism/relocation.rbs
145
- - sig/prism/serialize.rbs
146
- - sig/prism/string_query.rbs
147
- - sig/prism/visitor.rbs
182
+ - rbi/rubyvm/node_find.rbi
183
+ - sig/generated/prism.rbs
184
+ - sig/generated/prism/compiler.rbs
185
+ - sig/generated/prism/desugar_compiler.rbs
186
+ - sig/generated/prism/dispatcher.rbs
187
+ - sig/generated/prism/dot_visitor.rbs
188
+ - sig/generated/prism/dsl.rbs
189
+ - sig/generated/prism/inspect_visitor.rbs
190
+ - sig/generated/prism/lex_compat.rbs
191
+ - sig/generated/prism/mutation_compiler.rbs
192
+ - sig/generated/prism/node.rbs
193
+ - sig/generated/prism/node_ext.rbs
194
+ - sig/generated/prism/node_find.rbs
195
+ - sig/generated/prism/parse_result.rbs
196
+ - sig/generated/prism/parse_result/errors.rbs
197
+ - sig/generated/prism/parse_result/newlines.rbs
198
+ - sig/generated/prism/pattern.rbs
199
+ - sig/generated/prism/reflection.rbs
200
+ - sig/generated/prism/relocation.rbs
201
+ - sig/generated/prism/serialize.rbs
202
+ - sig/generated/prism/string_query.rbs
203
+ - sig/generated/prism/translation.rbs
204
+ - sig/generated/prism/visitor.rbs
205
+ - sig/manifest.yaml
206
+ - src/arena.c
207
+ - src/buffer.c
208
+ - src/char.c
209
+ - src/comments.c
210
+ - src/constant_pool.c
148
211
  - src/diagnostic.c
149
212
  - src/encoding.c
213
+ - src/errors_format.c
214
+ - src/integer.c
215
+ - src/json.c
216
+ - src/line_offset_list.c
217
+ - src/list.c
218
+ - src/memchr.c
150
219
  - src/node.c
151
220
  - src/options.c
152
- - src/pack.c
221
+ - src/parser.c
153
222
  - src/prettyprint.c
154
223
  - src/prism.c
155
224
  - src/regexp.c
156
225
  - src/serialize.c
226
+ - src/source.c
157
227
  - src/static_literals.c
158
- - src/token_type.c
159
- - src/util/pm_buffer.c
160
- - src/util/pm_char.c
161
- - src/util/pm_constant_pool.c
162
- - src/util/pm_integer.c
163
- - src/util/pm_list.c
164
- - src/util/pm_memchr.c
165
- - src/util/pm_newline_list.c
166
- - src/util/pm_string.c
167
- - src/util/pm_strncasecmp.c
168
- - src/util/pm_strpbrk.c
228
+ - src/string_query.c
229
+ - src/stringy.c
230
+ - src/strncasecmp.c
231
+ - src/strpbrk.c
232
+ - src/tokens.c
233
+ - src/xxh3.c
169
234
  homepage: https://github.com/ruby/prism
170
235
  licenses:
171
236
  - MIT
@@ -187,7 +252,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
187
252
  - !ruby/object:Gem::Version
188
253
  version: '0'
189
254
  requirements: []
190
- rubygems_version: 3.6.9
255
+ rubygems_version: 4.0.20
191
256
  specification_version: 4
192
257
  summary: Prism Ruby parser
193
258
  test_files: []