prism 1.9.0 → 2.0.0.rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +88 -0
- data/CODE_OF_CONDUCT.md +1 -1
- data/LICENSE.md +1 -1
- data/Makefile +41 -12
- data/README.md +35 -97
- data/config.yml +174 -161
- data/docs/build_system.md +2 -2
- data/docs/configuration.md +5 -4
- data/docs/encoding.md +2 -2
- data/docs/fuzzing.md +4 -11
- data/docs/javascript.md +19 -4
- data/docs/mapping.md +1 -1
- data/docs/releasing.md +77 -5
- data/docs/relocation.md +2 -4
- data/docs/serialization.md +22 -24
- data/docs/testing.md +46 -29
- data/ext/prism/api_node.c +1461 -1401
- data/ext/prism/extconf.rb +3 -4
- data/ext/prism/extension.c +796 -422
- data/ext/prism/extension.h +3 -3
- data/include/prism/arena.h +37 -0
- data/include/prism/ast.h +2727 -610
- data/include/prism/buffer.h +52 -0
- data/include/prism/comments.h +55 -0
- data/include/prism/compiler/accel.h +26 -0
- data/include/prism/compiler/align.h +36 -0
- data/include/prism/compiler/assume.h +25 -0
- data/include/prism/compiler/bswap.h +46 -0
- data/include/prism/compiler/endianness.h +34 -0
- data/include/prism/compiler/exported.h +24 -0
- data/include/prism/compiler/fallthrough.h +22 -0
- data/include/prism/compiler/filesystem.h +32 -0
- data/include/prism/compiler/flex_array.h +19 -0
- data/include/prism/compiler/force_inline.h +21 -0
- data/include/prism/compiler/format.h +25 -0
- data/include/prism/compiler/inline.h +17 -0
- data/include/prism/compiler/nodiscard.h +22 -0
- data/include/prism/compiler/nonnull.h +18 -0
- data/include/prism/compiler/unused.h +18 -0
- data/include/prism/constant_pool.h +81 -0
- data/include/prism/diagnostic.h +38 -403
- data/include/prism/errors_format.h +53 -0
- data/include/prism/excludes.h +29 -0
- data/include/prism/integer.h +41 -0
- data/include/prism/internal/allocator.h +68 -0
- data/include/prism/internal/allocator_debug.h +88 -0
- data/include/prism/internal/arena.h +108 -0
- data/include/prism/internal/bit.h +42 -0
- data/include/prism/internal/buffer.h +94 -0
- data/include/prism/internal/char.h +145 -0
- data/include/prism/internal/comments.h +53 -0
- data/include/prism/internal/constant_pool.h +152 -0
- data/include/prism/internal/diagnostic.h +389 -0
- data/include/prism/{encoding.h → internal/encoding.h} +27 -68
- data/include/prism/internal/integer.h +68 -0
- data/include/prism/internal/isinf.h +16 -0
- data/include/prism/internal/line_offset_list.h +34 -0
- data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
- data/include/prism/internal/magic_comments.h +23 -0
- data/include/prism/internal/memchr.h +15 -0
- data/include/prism/internal/node.h +38 -0
- data/include/prism/internal/options.h +232 -0
- data/include/prism/internal/parser.h +1004 -0
- data/include/prism/internal/regexp.h +26 -0
- data/include/prism/internal/serialize.h +39 -0
- data/include/prism/internal/source.h +72 -0
- data/include/prism/{static_literals.h → internal/static_literals.h} +26 -47
- data/include/prism/internal/stringy.h +35 -0
- data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
- data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
- data/include/prism/internal/tokens.h +11 -0
- data/include/prism/internal/xxh3.h +16 -0
- data/include/prism/json.h +32 -0
- data/include/prism/line_offset_list.h +61 -0
- data/include/prism/magic_comments.h +35 -0
- data/include/prism/node.h +18 -53
- data/include/prism/options.h +146 -280
- data/include/prism/parser.h +356 -867
- data/include/prism/prettyprint.h +6 -9
- data/include/prism/serialize.h +115 -0
- data/include/prism/source.h +155 -0
- data/include/prism/stream.h +28 -0
- data/include/prism/string_query.h +74 -0
- data/include/prism/stringy.h +72 -0
- data/include/prism/version.h +12 -3
- data/include/prism.h +43 -320
- data/lib/prism/compiler.rb +319 -305
- data/lib/prism/desugar_compiler.rb +105 -34
- data/lib/prism/dispatcher.rb +646 -473
- data/lib/prism/dot_visitor.rb +592 -511
- data/lib/prism/dsl.rb +391 -25
- data/lib/prism/ffi.rb +258 -90
- data/lib/prism/inspect_visitor.rb +357 -325
- data/lib/prism/lex_compat.rb +254 -212
- data/lib/prism/mutation_compiler.rb +313 -305
- data/lib/prism/node.rb +20966 -6992
- data/lib/prism/node_ext.rb +180 -187
- data/lib/prism/node_find.rb +215 -0
- data/lib/prism/parse_result/errors.rb +7 -1
- data/lib/prism/parse_result/newlines.rb +276 -8
- data/lib/prism/parse_result.rb +468 -139
- data/lib/prism/pattern.rb +65 -20
- data/lib/prism/reflection.rb +23 -12
- data/lib/prism/relocation.rb +149 -87
- data/lib/prism/serialize.rb +3460 -539
- data/lib/prism/string_query.rb +24 -1
- data/lib/prism/translation/parser/builder.rb +10 -2
- data/lib/prism/translation/parser/compiler.rb +44 -52
- data/lib/prism/translation/parser/lexer.rb +41 -92
- data/lib/prism/translation/parser.rb +1 -1
- data/lib/prism/translation/parser_current.rb +1 -1
- data/lib/prism/translation/ripper/lexer.rb +8 -10
- data/lib/prism/translation/ripper/sexp.rb +2 -10
- data/lib/prism/translation/ripper/shim.rb +2 -0
- data/lib/prism/translation/ripper.rb +967 -209
- data/lib/prism/translation/ruby_parser.rb +73 -316
- data/lib/prism/translation.rb +2 -0
- data/lib/prism/visitor.rb +632 -5
- data/lib/prism.rb +79 -9
- data/prism.gemspec +128 -62
- data/rbi/generated/prism/compiler.rbi +489 -0
- data/rbi/generated/prism/desugar_compiler.rbi +306 -0
- data/rbi/generated/prism/dispatcher.rbi +1138 -0
- data/rbi/generated/prism/dot_visitor.rbi +618 -0
- data/rbi/generated/prism/dsl.rbi +751 -0
- data/rbi/generated/prism/inspect_visitor.rbi +503 -0
- data/rbi/generated/prism/lex_compat.rbi +162 -0
- data/rbi/generated/prism/mutation_compiler.rbi +464 -0
- data/rbi/generated/prism/node.rbi +14407 -0
- data/rbi/generated/prism/node_ext.rbi +259 -0
- data/rbi/generated/prism/node_find.rbi +79 -0
- data/rbi/generated/prism/parse_result/errors.rbi +21 -0
- data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
- data/rbi/generated/prism/parse_result.rbi +776 -0
- data/rbi/generated/prism/pattern.rbi +134 -0
- data/rbi/generated/prism/reflection.rbi +89 -0
- data/rbi/generated/prism/relocation.rbi +329 -0
- data/rbi/generated/prism/serialize.rbi +184 -0
- data/rbi/generated/prism/string_query.rbi +43 -0
- data/rbi/generated/prism/translation.rbi +7 -0
- data/rbi/generated/prism/visitor.rbi +650 -0
- data/rbi/generated/prism.rbi +86 -0
- data/rbi/prism/translation/parser_versions.rbi +5 -0
- data/rbi/rubyvm/node_find.rbi +14 -0
- data/sig/generated/prism/compiler.rbs +492 -0
- data/sig/generated/prism/desugar_compiler.rbs +305 -0
- data/sig/generated/prism/dispatcher.rbs +1292 -0
- data/sig/generated/prism/dot_visitor.rbs +635 -0
- data/sig/generated/prism/dsl.rbs +928 -0
- data/sig/generated/prism/inspect_visitor.rbs +509 -0
- data/sig/generated/prism/lex_compat.rbs +166 -0
- data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
- data/sig/generated/prism/node.rbs +20896 -0
- data/sig/generated/prism/node_ext.rbs +290 -0
- data/sig/generated/prism/node_find.rbs +90 -0
- data/sig/generated/prism/parse_result/errors.rbs +22 -0
- data/sig/generated/prism/parse_result/newlines.rbs +223 -0
- data/sig/generated/prism/parse_result.rbs +871 -0
- data/sig/generated/prism/pattern.rbs +156 -0
- data/sig/generated/prism/reflection.rbs +101 -0
- data/sig/generated/prism/relocation.rbs +428 -0
- data/sig/generated/prism/serialize.rbs +198 -0
- data/sig/generated/prism/string_query.rbs +43 -0
- data/sig/generated/prism/translation.rbs +8 -0
- data/sig/generated/prism/visitor.rbs +1111 -0
- data/sig/generated/prism.rbs +84 -0
- data/sig/manifest.yaml +2 -0
- data/src/arena.c +117 -0
- data/src/{util/pm_buffer.c → buffer.c} +53 -21
- data/src/{util/pm_char.c → char.c} +28 -57
- data/src/comments.c +454 -0
- data/src/constant_pool.c +406 -0
- data/src/diagnostic.c +97 -65
- data/src/encoding.c +8 -3
- data/src/errors_format.c +519 -0
- data/src/{util/pm_integer.c → integer.c} +33 -22
- data/src/json.c +5733 -0
- data/src/line_offset_list.c +100 -0
- data/src/list.c +24 -0
- data/src/{util/pm_memchr.c → memchr.c} +8 -6
- data/src/node.c +3504 -7674
- data/src/options.c +162 -49
- data/src/parser.c +367 -0
- data/src/prettyprint.c +421 -354
- data/src/prism.c +8402 -7161
- data/src/regexp.c +961 -45
- data/src/serialize.c +460 -424
- data/src/source.c +518 -0
- data/src/static_literals.c +111 -43
- data/src/string_query.c +180 -0
- data/src/stringy.c +91 -0
- data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
- data/src/strpbrk.c +447 -0
- data/src/{token_type.c → tokens.c} +39 -18
- data/src/xxh3.c +292 -0
- metadata +128 -63
- data/ext/prism/api_pack.c +0 -276
- data/include/prism/defines.h +0 -260
- data/include/prism/pack.h +0 -163
- data/include/prism/regexp.h +0 -43
- data/include/prism/util/pm_buffer.h +0 -236
- data/include/prism/util/pm_char.h +0 -204
- data/include/prism/util/pm_constant_pool.h +0 -218
- data/include/prism/util/pm_integer.h +0 -130
- data/include/prism/util/pm_memchr.h +0 -29
- data/include/prism/util/pm_newline_list.h +0 -113
- data/include/prism/util/pm_string.h +0 -200
- data/lib/prism/pack.rb +0 -230
- data/lib/prism/parse_result/comments.rb +0 -188
- data/rbi/prism/compiler.rbi +0 -12
- data/rbi/prism/dsl.rbi +0 -524
- data/rbi/prism/inspect_visitor.rbi +0 -12
- data/rbi/prism/node.rbi +0 -8750
- data/rbi/prism/node_ext.rbi +0 -107
- data/rbi/prism/parse_result.rbi +0 -404
- data/rbi/prism/reflection.rbi +0 -58
- data/rbi/prism/string_query.rbi +0 -12
- data/rbi/prism/visitor.rbi +0 -473
- data/rbi/prism.rbi +0 -63
- data/sig/prism/compiler.rbs +0 -9
- data/sig/prism/dispatcher.rbs +0 -19
- data/sig/prism/dot_visitor.rbs +0 -6
- data/sig/prism/dsl.rbs +0 -351
- data/sig/prism/inspect_visitor.rbs +0 -22
- data/sig/prism/lex_compat.rbs +0 -10
- data/sig/prism/node.rbs +0 -4042
- data/sig/prism/node_ext.rbs +0 -149
- data/sig/prism/pack.rbs +0 -43
- data/sig/prism/parse_result/comments.rbs +0 -38
- data/sig/prism/parse_result.rbs +0 -197
- data/sig/prism/pattern.rbs +0 -13
- data/sig/prism/reflection.rbs +0 -50
- data/sig/prism/relocation.rbs +0 -185
- data/sig/prism/serialize.rbs +0 -8
- data/sig/prism/string_query.rbs +0 -11
- data/sig/prism/visitor.rbs +0 -169
- data/sig/prism.rbs +0 -272
- data/src/pack.c +0 -509
- data/src/util/pm_constant_pool.c +0 -342
- data/src/util/pm_list.c +0 -49
- data/src/util/pm_newline_list.c +0 -125
- data/src/util/pm_string.c +0 -381
- data/src/util/pm_strpbrk.c +0 -206
data/src/xxh3.c
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An implementation of the XXH3 hash function.
|
|
3
|
+
*
|
|
4
|
+
* The reason some of these names do not look quite like the style of the rest
|
|
5
|
+
* of the code in Prism is that this was adapted straight from the XXH3
|
|
6
|
+
* specification here:
|
|
7
|
+
*
|
|
8
|
+
* https://github.com/Cyan4973/xxHash/blob/release/doc/xxhash_spec.md
|
|
9
|
+
*
|
|
10
|
+
* To better understand this code it is recommended to read the XXH3
|
|
11
|
+
* specification linked above, where you should be able to follow along with
|
|
12
|
+
* this implementation more easily. Note this only implements the XXH3_64
|
|
13
|
+
* variant with the default secret and a seed of 0. Additionally it does not
|
|
14
|
+
* include a lot of the various CPU-specific optimizations that exist in the
|
|
15
|
+
* reference implementation.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
#include "prism/internal/xxh3.h"
|
|
19
|
+
|
|
20
|
+
#include "prism/compiler/accel.h"
|
|
21
|
+
#include "prism/compiler/bswap.h"
|
|
22
|
+
#include "prism/compiler/endianness.h"
|
|
23
|
+
|
|
24
|
+
#include <string.h>
|
|
25
|
+
|
|
26
|
+
#if defined(PRISM_HAS_NEON) && defined(PRISM_LITTLE_ENDIAN_COMPTIME)
|
|
27
|
+
#include <arm_neon.h>
|
|
28
|
+
#define XXH3_NEON
|
|
29
|
+
#elif defined(PRISM_HAS_SSE2)
|
|
30
|
+
#include <emmintrin.h>
|
|
31
|
+
#define XXH3_SSE2
|
|
32
|
+
#endif
|
|
33
|
+
|
|
34
|
+
#if defined(__SIZEOF_INT128__)
|
|
35
|
+
static uint64_t
|
|
36
|
+
XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
|
|
37
|
+
__uint128_t x = (__uint128_t) lhs * (__uint128_t) rhs;
|
|
38
|
+
return ((uint64_t) (x & 0xFFFFFFFFFFFFFFFFULL)) ^ ((uint64_t) (x >> 64));
|
|
39
|
+
}
|
|
40
|
+
#else
|
|
41
|
+
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64))
|
|
42
|
+
#include <intrin.h>
|
|
43
|
+
#endif
|
|
44
|
+
|
|
45
|
+
static uint64_t
|
|
46
|
+
XXH3_mul128_fold64(uint64_t lhs, uint64_t rhs) {
|
|
47
|
+
uint64_t lo, hi;
|
|
48
|
+
#if defined(_MSC_VER) && defined(_M_X64)
|
|
49
|
+
lo = _umul128(lhs, rhs, &hi);
|
|
50
|
+
#elif defined(_MSC_VER) && defined(_M_ARM64)
|
|
51
|
+
hi = __umulh(lhs, rhs);
|
|
52
|
+
lo = lhs * rhs;
|
|
53
|
+
#else
|
|
54
|
+
uint64_t lo_lo = (lhs & 0xFFFFFFFF) * (rhs & 0xFFFFFFFF);
|
|
55
|
+
uint64_t hi_lo = (lhs >> 32) * (rhs & 0xFFFFFFFF);
|
|
56
|
+
uint64_t lo_hi = (lhs & 0xFFFFFFFF) * (rhs >> 32);
|
|
57
|
+
uint64_t hi_hi = (lhs >> 32) * (rhs >> 32);
|
|
58
|
+
|
|
59
|
+
uint64_t cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
|
|
60
|
+
hi = hi_hi + (hi_lo >> 32) + (cross >> 32);
|
|
61
|
+
lo = (cross << 32) | (lo_lo & 0xFFFFFFFF);
|
|
62
|
+
#endif
|
|
63
|
+
return lo ^ hi;
|
|
64
|
+
}
|
|
65
|
+
#endif
|
|
66
|
+
|
|
67
|
+
static uint32_t
|
|
68
|
+
read32(const void* ptr) {
|
|
69
|
+
uint32_t val;
|
|
70
|
+
memcpy(&val, ptr, sizeof(val));
|
|
71
|
+
if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
|
|
72
|
+
val = pm_bswap32(val);
|
|
73
|
+
}
|
|
74
|
+
return val;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
static uint64_t
|
|
78
|
+
read64(const void* ptr) {
|
|
79
|
+
uint64_t val;
|
|
80
|
+
memcpy(&val, ptr, sizeof(val));
|
|
81
|
+
if (!PRISM_LITTLE_ENDIAN_RUNTIME) {
|
|
82
|
+
val = pm_bswap64(val);
|
|
83
|
+
}
|
|
84
|
+
return val;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
static const uint64_t PRIME32_1 = 0x9E3779B1U;
|
|
88
|
+
static const uint64_t PRIME32_2 = 0x85EBCA77U;
|
|
89
|
+
static const uint64_t PRIME32_3 = 0xC2B2AE3DU;
|
|
90
|
+
static const uint64_t PRIME64_1 = 0x9E3779B185EBCA87ULL;
|
|
91
|
+
static const uint64_t PRIME64_2 = 0xC2B2AE3D27D4EB4FULL;
|
|
92
|
+
static const uint64_t PRIME64_3 = 0x165667B19E3779F9ULL;
|
|
93
|
+
static const uint64_t PRIME64_4 = 0x85EBCA77C2B2AE63ULL;
|
|
94
|
+
static const uint64_t PRIME64_5 = 0x27D4EB2F165667C5ULL;
|
|
95
|
+
static const uint64_t PRIME_MX1 = 0x165667919E3779F9ULL;
|
|
96
|
+
static const uint64_t PRIME_MX2 = 0x9FB21C651E98DF25ULL;
|
|
97
|
+
|
|
98
|
+
static const uint8_t secret[192] = {
|
|
99
|
+
0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
|
|
100
|
+
0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
|
|
101
|
+
0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
|
|
102
|
+
0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
|
|
103
|
+
0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
|
|
104
|
+
0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
|
|
105
|
+
0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
|
|
106
|
+
0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
|
|
107
|
+
0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
|
|
108
|
+
0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
|
|
109
|
+
0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
|
|
110
|
+
0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
static uint64_t
|
|
114
|
+
avalanche(uint64_t x) {
|
|
115
|
+
x ^= (x >> 37);
|
|
116
|
+
x *= PRIME_MX1;
|
|
117
|
+
x ^= (x >> 32);
|
|
118
|
+
return x;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
static uint64_t
|
|
122
|
+
avalanche_XXH64(uint64_t x) {
|
|
123
|
+
x ^= (x >> 33);
|
|
124
|
+
x *= PRIME64_2;
|
|
125
|
+
x ^= (x >> 29);
|
|
126
|
+
x *= PRIME64_3;
|
|
127
|
+
x ^= (x >> 32);
|
|
128
|
+
return x;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
static uint64_t
|
|
132
|
+
XXH3_64_empty(void) {
|
|
133
|
+
return avalanche_XXH64(read64(secret + 56) ^ read64(secret + 64));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
static uint64_t
|
|
137
|
+
XXH3_64_1to3(const uint8_t *input, size_t length) {
|
|
138
|
+
uint32_t combined = (uint32_t) input[length - 1] | ((uint32_t) length << 8) | ((uint32_t) input[0] << 16) | ((uint32_t) input[length >> 1] << 24);
|
|
139
|
+
return avalanche_XXH64((uint64_t) (read32(secret) ^ read32(secret + 4)) ^ (uint64_t) combined);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
static uint64_t
|
|
143
|
+
XXH3_64_4to8(const uint8_t *input, size_t length) {
|
|
144
|
+
#define rot(x_, r_) (((x_) << (r_)) | ((x_) >> (64 - (r_))))
|
|
145
|
+
|
|
146
|
+
uint64_t combined = (uint64_t) read32(input + length - 4) | ((uint64_t) read32(input) << 32);
|
|
147
|
+
uint64_t value = (read64(secret + 8) ^ read64(secret + 16)) ^ combined;
|
|
148
|
+
value ^= rot(value, 49) ^ rot(value, 24);
|
|
149
|
+
value *= PRIME_MX2;
|
|
150
|
+
value ^= ((value >> 35) + length);
|
|
151
|
+
value *= PRIME_MX2;
|
|
152
|
+
value ^= (value >> 28);
|
|
153
|
+
return value;
|
|
154
|
+
|
|
155
|
+
#undef rot
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
static uint64_t
|
|
159
|
+
XXH3_64_9to16(const uint8_t *input, size_t length) {
|
|
160
|
+
uint64_t lo = (read64(secret + 24) ^ read64(secret + 32)) ^ read64(input);
|
|
161
|
+
uint64_t hi = (read64(secret + 40) ^ read64(secret + 48)) ^ read64(input + length - 8);
|
|
162
|
+
return avalanche(length + pm_bswap64(lo) + hi + XXH3_mul128_fold64(lo, hi));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
static uint64_t
|
|
166
|
+
XXH3_64_17to240(const uint8_t *input, size_t length) {
|
|
167
|
+
#define mix(data_, offset_) XXH3_mul128_fold64(read64((data_)) ^ read64(secret + (offset_)), read64((data_) + 8) ^ read64(secret + (offset_) + 8))
|
|
168
|
+
|
|
169
|
+
uint64_t acc = length * PRIME64_1;
|
|
170
|
+
|
|
171
|
+
if (length <= 128) {
|
|
172
|
+
size_t idx = (length - 1) / 32;
|
|
173
|
+
do {
|
|
174
|
+
acc += mix(input + idx * 16, idx * 32) + mix(input + length - idx * 16 - 16, idx * 32 + 16);
|
|
175
|
+
} while (idx--);
|
|
176
|
+
} else {
|
|
177
|
+
for (size_t idx = 0; idx < 8; idx++) {
|
|
178
|
+
acc += mix(input + idx * 16, idx * 16);
|
|
179
|
+
}
|
|
180
|
+
acc = avalanche(acc);
|
|
181
|
+
for (size_t idx = 8; idx < length >> 4; idx++) {
|
|
182
|
+
acc += mix(input + idx * 16, (idx - 8) * 16 + 3);
|
|
183
|
+
}
|
|
184
|
+
acc += mix(input + length - 16, 119);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
return avalanche(acc);
|
|
188
|
+
|
|
189
|
+
#undef mix
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
static void
|
|
193
|
+
accumulate(uint64_t acc[8], const uint8_t *stripe, size_t secret_offset) {
|
|
194
|
+
#if defined(XXH3_NEON)
|
|
195
|
+
for (int idx = 0; idx < 8; idx += 2) {
|
|
196
|
+
uint64x2_t data = vreinterpretq_u64_u8(vld1q_u8(stripe + 8 * idx));
|
|
197
|
+
uint64x2_t value = veorq_u64(data, vreinterpretq_u64_u8(vld1q_u8(secret + secret_offset + 8 * idx)));
|
|
198
|
+
uint64x2_t product = vmull_u32(vmovn_u64(value), vshrn_n_u64(value, 32));
|
|
199
|
+
vst1q_u64(acc + idx, vaddq_u64(vld1q_u64(acc + idx), vaddq_u64(vextq_u64(data, data, 1), product)));
|
|
200
|
+
}
|
|
201
|
+
#elif defined(XXH3_SSE2)
|
|
202
|
+
for (int idx = 0; idx < 8; idx += 2) {
|
|
203
|
+
__m128i data = _mm_loadu_si128((const __m128i *) (stripe + 8 * idx));
|
|
204
|
+
__m128i value = _mm_xor_si128(data, _mm_loadu_si128((const __m128i *) (secret + secret_offset + 8 * idx)));
|
|
205
|
+
__m128i product = _mm_mul_epu32(value, _mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)));
|
|
206
|
+
_mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_loadu_si128((const __m128i *) (acc + idx)), _mm_add_epi64(_mm_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2)), product)));
|
|
207
|
+
}
|
|
208
|
+
#else
|
|
209
|
+
for (int idx = 0; idx < 8; idx++) {
|
|
210
|
+
uint64_t data = read64(stripe + 8 * idx);
|
|
211
|
+
uint64_t value = data ^ read64(secret + secret_offset + 8 * idx);
|
|
212
|
+
acc[idx ^ 1] = acc[idx ^ 1] + data;
|
|
213
|
+
acc[idx] += (value & 0xFFFFFFFF) * (value >> 32);
|
|
214
|
+
}
|
|
215
|
+
#endif
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
static uint64_t
|
|
219
|
+
XXH3_64_large(const uint8_t *input, size_t length) {
|
|
220
|
+
const size_t secret_length = sizeof(secret);
|
|
221
|
+
const size_t stripes_per_block = (secret_length - 64) / 8;
|
|
222
|
+
const size_t block_size = 64 * stripes_per_block;
|
|
223
|
+
const size_t block_count = (length - 1) / block_size;
|
|
224
|
+
const size_t last_offset = block_count * block_size;
|
|
225
|
+
|
|
226
|
+
uint64_t acc[8] = {
|
|
227
|
+
PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3,
|
|
228
|
+
PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
for (size_t block_idx = 0; block_idx < block_count; block_idx++) {
|
|
232
|
+
const uint8_t *block = input + block_idx * block_size;
|
|
233
|
+
|
|
234
|
+
for (size_t stripe_idx = 0; stripe_idx < stripes_per_block; stripe_idx++) {
|
|
235
|
+
accumulate(acc, block + stripe_idx * 64, stripe_idx * 8);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
#if defined(XXH3_NEON)
|
|
239
|
+
const uint32x2_t prime = vdup_n_u32((uint32_t) PRIME32_1);
|
|
240
|
+
for (size_t idx = 0; idx < 8; idx += 2) {
|
|
241
|
+
uint64x2_t value = vld1q_u64(acc + idx);
|
|
242
|
+
value = veorq_u64(value, vshrq_n_u64(value, 47));
|
|
243
|
+
value = veorq_u64(value, vreinterpretq_u64_u8(vld1q_u8(secret + secret_length - 64 + idx * 8)));
|
|
244
|
+
vst1q_u64(acc + idx, vaddq_u64(vmull_u32(vmovn_u64(value), prime), vshlq_n_u64(vmull_u32(vshrn_n_u64(value, 32), prime), 32)));
|
|
245
|
+
}
|
|
246
|
+
#elif defined(XXH3_SSE2)
|
|
247
|
+
const __m128i prime = _mm_set1_epi32((int) (uint32_t) PRIME32_1);
|
|
248
|
+
for (size_t idx = 0; idx < 8; idx += 2) {
|
|
249
|
+
__m128i value = _mm_loadu_si128((const __m128i *) (acc + idx));
|
|
250
|
+
value = _mm_xor_si128(value, _mm_srli_epi64(value, 47));
|
|
251
|
+
value = _mm_xor_si128(value, _mm_loadu_si128((const __m128i *) (secret + secret_length - 64 + idx * 8)));
|
|
252
|
+
_mm_storeu_si128((__m128i *) (acc + idx), _mm_add_epi64(_mm_mul_epu32(value, prime), _mm_slli_epi64(_mm_mul_epu32(_mm_shuffle_epi32(value, _MM_SHUFFLE(0, 3, 0, 1)), prime), 32)));
|
|
253
|
+
}
|
|
254
|
+
#else
|
|
255
|
+
for (size_t idx = 0; idx < 8; idx++) {
|
|
256
|
+
acc[idx] ^= (acc[idx] >> 47);
|
|
257
|
+
acc[idx] ^= read64(secret + secret_length - 64 + idx * 8);
|
|
258
|
+
acc[idx] *= PRIME32_1;
|
|
259
|
+
}
|
|
260
|
+
#endif
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
for (size_t stripe_idx = 0; stripe_idx < (length - last_offset - 1) / 64; stripe_idx++) {
|
|
264
|
+
accumulate(acc, input + last_offset + stripe_idx * 64, stripe_idx * 8);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
accumulate(acc, input + length - 64, secret_length - 71);
|
|
268
|
+
|
|
269
|
+
uint64_t result = (uint64_t) length * PRIME64_1;
|
|
270
|
+
for (size_t idx = 0; idx < 4; idx++) {
|
|
271
|
+
result += XXH3_mul128_fold64(acc[idx * 2] ^ read64(secret + 11 + 16 * idx), acc[idx * 2 + 1] ^ read64(secret + 11 + 16 * idx + 8));
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
return avalanche(result);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
uint64_t
|
|
278
|
+
pm_xxh3(const uint8_t *input, size_t length) {
|
|
279
|
+
if (length == 0) {
|
|
280
|
+
return XXH3_64_empty();
|
|
281
|
+
} else if (length <= 3) {
|
|
282
|
+
return XXH3_64_1to3(input, length);
|
|
283
|
+
} else if (length <= 8) {
|
|
284
|
+
return XXH3_64_4to8(input, length);
|
|
285
|
+
} else if (length <= 16) {
|
|
286
|
+
return XXH3_64_9to16(input, length);
|
|
287
|
+
} else if (length <= 240) {
|
|
288
|
+
return XXH3_64_17to240(input, length);
|
|
289
|
+
} else {
|
|
290
|
+
return XXH3_64_large(input, length);
|
|
291
|
+
}
|
|
292
|
+
}
|
metadata
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: prism
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 2.0.0.rc2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
|
-
-
|
|
7
|
+
- Ruby developers
|
|
8
8
|
bindir: bin
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies: []
|
|
12
12
|
email:
|
|
13
|
-
- ruby@
|
|
13
|
+
- ruby-core@ruby-lang.org
|
|
14
14
|
executables: []
|
|
15
15
|
extensions:
|
|
16
16
|
- ext/prism/extconf.rb
|
|
@@ -44,32 +44,74 @@ files:
|
|
|
44
44
|
- docs/serialization.md
|
|
45
45
|
- docs/testing.md
|
|
46
46
|
- ext/prism/api_node.c
|
|
47
|
-
- ext/prism/api_pack.c
|
|
48
47
|
- ext/prism/extconf.rb
|
|
49
48
|
- ext/prism/extension.c
|
|
50
49
|
- ext/prism/extension.h
|
|
51
50
|
- include/prism.h
|
|
51
|
+
- include/prism/arena.h
|
|
52
52
|
- include/prism/ast.h
|
|
53
|
-
- include/prism/
|
|
53
|
+
- include/prism/buffer.h
|
|
54
|
+
- include/prism/comments.h
|
|
55
|
+
- include/prism/compiler/accel.h
|
|
56
|
+
- include/prism/compiler/align.h
|
|
57
|
+
- include/prism/compiler/assume.h
|
|
58
|
+
- include/prism/compiler/bswap.h
|
|
59
|
+
- include/prism/compiler/endianness.h
|
|
60
|
+
- include/prism/compiler/exported.h
|
|
61
|
+
- include/prism/compiler/fallthrough.h
|
|
62
|
+
- include/prism/compiler/filesystem.h
|
|
63
|
+
- include/prism/compiler/flex_array.h
|
|
64
|
+
- include/prism/compiler/force_inline.h
|
|
65
|
+
- include/prism/compiler/format.h
|
|
66
|
+
- include/prism/compiler/inline.h
|
|
67
|
+
- include/prism/compiler/nodiscard.h
|
|
68
|
+
- include/prism/compiler/nonnull.h
|
|
69
|
+
- include/prism/compiler/unused.h
|
|
70
|
+
- include/prism/constant_pool.h
|
|
54
71
|
- include/prism/diagnostic.h
|
|
55
|
-
- include/prism/
|
|
72
|
+
- include/prism/errors_format.h
|
|
73
|
+
- include/prism/excludes.h
|
|
74
|
+
- include/prism/integer.h
|
|
75
|
+
- include/prism/internal/allocator.h
|
|
76
|
+
- include/prism/internal/allocator_debug.h
|
|
77
|
+
- include/prism/internal/arena.h
|
|
78
|
+
- include/prism/internal/bit.h
|
|
79
|
+
- include/prism/internal/buffer.h
|
|
80
|
+
- include/prism/internal/char.h
|
|
81
|
+
- include/prism/internal/comments.h
|
|
82
|
+
- include/prism/internal/constant_pool.h
|
|
83
|
+
- include/prism/internal/diagnostic.h
|
|
84
|
+
- include/prism/internal/encoding.h
|
|
85
|
+
- include/prism/internal/integer.h
|
|
86
|
+
- include/prism/internal/isinf.h
|
|
87
|
+
- include/prism/internal/line_offset_list.h
|
|
88
|
+
- include/prism/internal/list.h
|
|
89
|
+
- include/prism/internal/magic_comments.h
|
|
90
|
+
- include/prism/internal/memchr.h
|
|
91
|
+
- include/prism/internal/node.h
|
|
92
|
+
- include/prism/internal/options.h
|
|
93
|
+
- include/prism/internal/parser.h
|
|
94
|
+
- include/prism/internal/regexp.h
|
|
95
|
+
- include/prism/internal/serialize.h
|
|
96
|
+
- include/prism/internal/source.h
|
|
97
|
+
- include/prism/internal/static_literals.h
|
|
98
|
+
- include/prism/internal/stringy.h
|
|
99
|
+
- include/prism/internal/strncasecmp.h
|
|
100
|
+
- include/prism/internal/strpbrk.h
|
|
101
|
+
- include/prism/internal/tokens.h
|
|
102
|
+
- include/prism/internal/xxh3.h
|
|
103
|
+
- include/prism/json.h
|
|
104
|
+
- include/prism/line_offset_list.h
|
|
105
|
+
- include/prism/magic_comments.h
|
|
56
106
|
- include/prism/node.h
|
|
57
107
|
- include/prism/options.h
|
|
58
|
-
- include/prism/pack.h
|
|
59
108
|
- include/prism/parser.h
|
|
60
109
|
- include/prism/prettyprint.h
|
|
61
|
-
- include/prism/
|
|
62
|
-
- include/prism/
|
|
63
|
-
- include/prism/
|
|
64
|
-
- include/prism/
|
|
65
|
-
- include/prism/
|
|
66
|
-
- include/prism/util/pm_integer.h
|
|
67
|
-
- include/prism/util/pm_list.h
|
|
68
|
-
- include/prism/util/pm_memchr.h
|
|
69
|
-
- include/prism/util/pm_newline_list.h
|
|
70
|
-
- include/prism/util/pm_string.h
|
|
71
|
-
- include/prism/util/pm_strncasecmp.h
|
|
72
|
-
- include/prism/util/pm_strpbrk.h
|
|
110
|
+
- include/prism/serialize.h
|
|
111
|
+
- include/prism/source.h
|
|
112
|
+
- include/prism/stream.h
|
|
113
|
+
- include/prism/string_query.h
|
|
114
|
+
- include/prism/stringy.h
|
|
73
115
|
- include/prism/version.h
|
|
74
116
|
- lib/prism.rb
|
|
75
117
|
- lib/prism/compiler.rb
|
|
@@ -83,9 +125,8 @@ files:
|
|
|
83
125
|
- lib/prism/mutation_compiler.rb
|
|
84
126
|
- lib/prism/node.rb
|
|
85
127
|
- lib/prism/node_ext.rb
|
|
86
|
-
- lib/prism/
|
|
128
|
+
- lib/prism/node_find.rb
|
|
87
129
|
- lib/prism/parse_result.rb
|
|
88
|
-
- lib/prism/parse_result/comments.rb
|
|
89
130
|
- lib/prism/parse_result/errors.rb
|
|
90
131
|
- lib/prism/parse_result/newlines.rb
|
|
91
132
|
- lib/prism/pattern.rb
|
|
@@ -113,59 +154,83 @@ files:
|
|
|
113
154
|
- lib/prism/translation/ruby_parser.rb
|
|
114
155
|
- lib/prism/visitor.rb
|
|
115
156
|
- prism.gemspec
|
|
116
|
-
- rbi/prism.rbi
|
|
117
|
-
- rbi/prism/compiler.rbi
|
|
118
|
-
- rbi/prism/
|
|
119
|
-
- rbi/prism/
|
|
120
|
-
- rbi/prism/
|
|
121
|
-
- rbi/prism/
|
|
122
|
-
- rbi/prism/
|
|
123
|
-
- rbi/prism/
|
|
124
|
-
- rbi/prism/
|
|
157
|
+
- rbi/generated/prism.rbi
|
|
158
|
+
- rbi/generated/prism/compiler.rbi
|
|
159
|
+
- rbi/generated/prism/desugar_compiler.rbi
|
|
160
|
+
- rbi/generated/prism/dispatcher.rbi
|
|
161
|
+
- rbi/generated/prism/dot_visitor.rbi
|
|
162
|
+
- rbi/generated/prism/dsl.rbi
|
|
163
|
+
- rbi/generated/prism/inspect_visitor.rbi
|
|
164
|
+
- rbi/generated/prism/lex_compat.rbi
|
|
165
|
+
- rbi/generated/prism/mutation_compiler.rbi
|
|
166
|
+
- rbi/generated/prism/node.rbi
|
|
167
|
+
- rbi/generated/prism/node_ext.rbi
|
|
168
|
+
- rbi/generated/prism/node_find.rbi
|
|
169
|
+
- rbi/generated/prism/parse_result.rbi
|
|
170
|
+
- rbi/generated/prism/parse_result/errors.rbi
|
|
171
|
+
- rbi/generated/prism/parse_result/newlines.rbi
|
|
172
|
+
- rbi/generated/prism/pattern.rbi
|
|
173
|
+
- rbi/generated/prism/reflection.rbi
|
|
174
|
+
- rbi/generated/prism/relocation.rbi
|
|
175
|
+
- rbi/generated/prism/serialize.rbi
|
|
176
|
+
- rbi/generated/prism/string_query.rbi
|
|
177
|
+
- rbi/generated/prism/translation.rbi
|
|
178
|
+
- rbi/generated/prism/visitor.rbi
|
|
125
179
|
- rbi/prism/translation/parser.rbi
|
|
126
180
|
- rbi/prism/translation/parser_versions.rbi
|
|
127
181
|
- rbi/prism/translation/ripper.rbi
|
|
128
|
-
- rbi/
|
|
129
|
-
- sig/prism.rbs
|
|
130
|
-
- sig/prism/compiler.rbs
|
|
131
|
-
- sig/prism/
|
|
132
|
-
- sig/prism/
|
|
133
|
-
- sig/prism/
|
|
134
|
-
- sig/prism/
|
|
135
|
-
- sig/prism/
|
|
136
|
-
- sig/prism/
|
|
137
|
-
- sig/prism/
|
|
138
|
-
- sig/prism/
|
|
139
|
-
- sig/prism/
|
|
140
|
-
- sig/prism/
|
|
141
|
-
- sig/prism/parse_result
|
|
142
|
-
- sig/prism/
|
|
143
|
-
- sig/prism/
|
|
144
|
-
- sig/prism/
|
|
145
|
-
- sig/prism/
|
|
146
|
-
- sig/prism/
|
|
147
|
-
- sig/prism/
|
|
182
|
+
- rbi/rubyvm/node_find.rbi
|
|
183
|
+
- sig/generated/prism.rbs
|
|
184
|
+
- sig/generated/prism/compiler.rbs
|
|
185
|
+
- sig/generated/prism/desugar_compiler.rbs
|
|
186
|
+
- sig/generated/prism/dispatcher.rbs
|
|
187
|
+
- sig/generated/prism/dot_visitor.rbs
|
|
188
|
+
- sig/generated/prism/dsl.rbs
|
|
189
|
+
- sig/generated/prism/inspect_visitor.rbs
|
|
190
|
+
- sig/generated/prism/lex_compat.rbs
|
|
191
|
+
- sig/generated/prism/mutation_compiler.rbs
|
|
192
|
+
- sig/generated/prism/node.rbs
|
|
193
|
+
- sig/generated/prism/node_ext.rbs
|
|
194
|
+
- sig/generated/prism/node_find.rbs
|
|
195
|
+
- sig/generated/prism/parse_result.rbs
|
|
196
|
+
- sig/generated/prism/parse_result/errors.rbs
|
|
197
|
+
- sig/generated/prism/parse_result/newlines.rbs
|
|
198
|
+
- sig/generated/prism/pattern.rbs
|
|
199
|
+
- sig/generated/prism/reflection.rbs
|
|
200
|
+
- sig/generated/prism/relocation.rbs
|
|
201
|
+
- sig/generated/prism/serialize.rbs
|
|
202
|
+
- sig/generated/prism/string_query.rbs
|
|
203
|
+
- sig/generated/prism/translation.rbs
|
|
204
|
+
- sig/generated/prism/visitor.rbs
|
|
205
|
+
- sig/manifest.yaml
|
|
206
|
+
- src/arena.c
|
|
207
|
+
- src/buffer.c
|
|
208
|
+
- src/char.c
|
|
209
|
+
- src/comments.c
|
|
210
|
+
- src/constant_pool.c
|
|
148
211
|
- src/diagnostic.c
|
|
149
212
|
- src/encoding.c
|
|
213
|
+
- src/errors_format.c
|
|
214
|
+
- src/integer.c
|
|
215
|
+
- src/json.c
|
|
216
|
+
- src/line_offset_list.c
|
|
217
|
+
- src/list.c
|
|
218
|
+
- src/memchr.c
|
|
150
219
|
- src/node.c
|
|
151
220
|
- src/options.c
|
|
152
|
-
- src/
|
|
221
|
+
- src/parser.c
|
|
153
222
|
- src/prettyprint.c
|
|
154
223
|
- src/prism.c
|
|
155
224
|
- src/regexp.c
|
|
156
225
|
- src/serialize.c
|
|
226
|
+
- src/source.c
|
|
157
227
|
- src/static_literals.c
|
|
158
|
-
- src/
|
|
159
|
-
- src/
|
|
160
|
-
- src/
|
|
161
|
-
- src/
|
|
162
|
-
- src/
|
|
163
|
-
- src/
|
|
164
|
-
- src/util/pm_memchr.c
|
|
165
|
-
- src/util/pm_newline_list.c
|
|
166
|
-
- src/util/pm_string.c
|
|
167
|
-
- src/util/pm_strncasecmp.c
|
|
168
|
-
- src/util/pm_strpbrk.c
|
|
228
|
+
- src/string_query.c
|
|
229
|
+
- src/stringy.c
|
|
230
|
+
- src/strncasecmp.c
|
|
231
|
+
- src/strpbrk.c
|
|
232
|
+
- src/tokens.c
|
|
233
|
+
- src/xxh3.c
|
|
169
234
|
homepage: https://github.com/ruby/prism
|
|
170
235
|
licenses:
|
|
171
236
|
- MIT
|
|
@@ -187,7 +252,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
187
252
|
- !ruby/object:Gem::Version
|
|
188
253
|
version: '0'
|
|
189
254
|
requirements: []
|
|
190
|
-
rubygems_version:
|
|
255
|
+
rubygems_version: 4.0.20
|
|
191
256
|
specification_version: 4
|
|
192
257
|
summary: Prism Ruby parser
|
|
193
258
|
test_files: []
|