deflate-ruby 1.0.1 → 1.0.2

data/ext/deflate_ruby/libdeflate/x86/cpu_features.c

@@ -0,0 +1,213 @@
+/*
+ * x86/cpu_features.c - feature detection for x86 CPUs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "../cpu_features_common.h" /* must be included first */
+#include "cpu_features.h"
+
+#ifdef X86_CPU_FEATURES_KNOWN
+/* Runtime x86 CPU feature detection is supported. */
+
+/* Execute the CPUID instruction. */
+static inline void
+cpuid(u32 leaf, u32 subleaf, u32 *a, u32 *b, u32 *c, u32 *d)
+{
+#ifdef _MSC_VER
+	int result[4];
+
+	__cpuidex(result, leaf, subleaf);
+	*a = result[0];
+	*b = result[1];
+	*c = result[2];
+	*d = result[3];
+#else
+	__asm__ volatile("cpuid" : "=a" (*a), "=b" (*b), "=c" (*c), "=d" (*d)
+			 : "a" (leaf), "c" (subleaf));
+#endif
+}
+
+/* Read an extended control register. */
+static inline u64
+read_xcr(u32 index)
+{
+#ifdef _MSC_VER
+	return _xgetbv(index);
+#else
+	u32 d, a;
+
+	/*
+	 * Execute the "xgetbv" instruction.  Old versions of binutils do not
+	 * recognize this instruction, so list the raw bytes instead.
+	 *
+	 * This must be 'volatile' to prevent this code from being moved out
+	 * from under the check for OSXSAVE.
+	 */
+	__asm__ volatile(".byte 0x0f, 0x01, 0xd0" :
+			 "=d" (d), "=a" (a) : "c" (index));
+
+	return ((u64)d << 32) | a;
+#endif
+}
+
+static const struct cpu_feature x86_cpu_feature_table[] = {
+	{X86_CPU_FEATURE_SSE2,		"sse2"},
+	{X86_CPU_FEATURE_PCLMULQDQ,	"pclmulqdq"},
+	{X86_CPU_FEATURE_AVX,		"avx"},
+	{X86_CPU_FEATURE_AVX2,		"avx2"},
+	{X86_CPU_FEATURE_BMI2,		"bmi2"},
+	{X86_CPU_FEATURE_ZMM,		"zmm"},
+	{X86_CPU_FEATURE_AVX512BW,	"avx512bw"},
+	{X86_CPU_FEATURE_AVX512VL,	"avx512vl"},
+	{X86_CPU_FEATURE_VPCLMULQDQ,	"vpclmulqdq"},
+	{X86_CPU_FEATURE_AVX512VNNI,	"avx512_vnni"},
+	{X86_CPU_FEATURE_AVXVNNI,	"avx_vnni"},
+};
+
+volatile u32 libdeflate_x86_cpu_features = 0;
+
+static inline bool
+os_supports_avx512(u64 xcr0)
+{
+#ifdef __APPLE__
+	/*
+	 * The Darwin kernel had a bug where it could corrupt the opmask
+	 * registers.  See
+	 * https://community.intel.com/t5/Software-Tuning-Performance/MacOS-Darwin-kernel-bug-clobbers-AVX-512-opmask-register-state/m-p/1327259
+	 * Darwin also does not initially set the XCR0 bits for AVX512, but they
+	 * are set if the thread tries to use AVX512 anyway.  Thus, to safely
+	 * and consistently use AVX512 on macOS we'd need to check the kernel
+	 * version as well as detect AVX512 support using a macOS-specific
+	 * method.  We don't bother with this, especially given Apple's
+	 * transition to arm64.
+	 */
+	return false;
+#else
+	return (xcr0 & 0xe6) == 0xe6;
+#endif
+}
+
+/*
+ * Don't use 512-bit vectors (ZMM registers) on Intel CPUs before Rocket Lake
+ * and Sapphire Rapids, due to the overly-eager downclocking which can reduce
+ * the performance of workloads that use ZMM registers only occasionally.
+ */
+static inline bool
+allow_512bit_vectors(const u32 manufacturer[3], u32 family, u32 model)
+{
+#ifdef TEST_SUPPORT__DO_NOT_USE
+	return true;
+#endif
+	if (memcmp(manufacturer, "GenuineIntel", 12) != 0)
+		return true;
+	if (family != 6)
+		return true;
+	switch (model) {
+	case 85: /* Skylake (Server), Cascade Lake, Cooper Lake */
+	case 106: /* Ice Lake (Server) */
+	case 108: /* Ice Lake (Server) */
+	case 126: /* Ice Lake (Client) */
+	case 140: /* Tiger Lake */
+	case 141: /* Tiger Lake */
+		return false;
+	}
+	return true;
+}
+
+/* Initialize libdeflate_x86_cpu_features. */
+void libdeflate_init_x86_cpu_features(void)
+{
+	u32 max_leaf;
+	u32 manufacturer[3];
+	u32 family, model;
+	u32 a, b, c, d;
+	u64 xcr0 = 0;
+	u32 features = 0;
+
+	/* EAX=0: Highest Function Parameter and Manufacturer ID */
+	cpuid(0, 0, &max_leaf, &manufacturer[0], &manufacturer[2],
+	      &manufacturer[1]);
+	if (max_leaf < 1)
+		goto out;
+
+	/* EAX=1: Processor Info and Feature Bits */
+	cpuid(1, 0, &a, &b, &c, &d);
+	family = (a >> 8) & 0xf;
+	model = (a >> 4) & 0xf;
+	if (family == 6 || family == 0xf)
+		model += (a >> 12) & 0xf0;
+	if (family == 0xf)
+		family += (a >> 20) & 0xff;
+	if (d & (1 << 26))
+		features |= X86_CPU_FEATURE_SSE2;
+	/*
+	 * No known CPUs have pclmulqdq without sse4.1, so in practice code
+	 * targeting pclmulqdq can use sse4.1 instructions.  But to be safe,
+	 * explicitly check for both the pclmulqdq and sse4.1 bits.
+	 */
+	if ((c & (1 << 1)) && (c & (1 << 19)))
+		features |= X86_CPU_FEATURE_PCLMULQDQ;
+	if (c & (1 << 27))
+		xcr0 = read_xcr(0);
+	if ((c & (1 << 28)) && ((xcr0 & 0x6) == 0x6))
+		features |= X86_CPU_FEATURE_AVX;
+
+	if (max_leaf < 7)
+		goto out;
+
+	/* EAX=7, ECX=0: Extended Features */
+	cpuid(7, 0, &a, &b, &c, &d);
+	if (b & (1 << 8))
+		features |= X86_CPU_FEATURE_BMI2;
+	if ((xcr0 & 0x6) == 0x6) {
+		if (b & (1 << 5))
+			features |= X86_CPU_FEATURE_AVX2;
+		if (c & (1 << 10))
+			features |= X86_CPU_FEATURE_VPCLMULQDQ;
+	}
+	if (os_supports_avx512(xcr0)) {
+		if (allow_512bit_vectors(manufacturer, family, model))
+			features |= X86_CPU_FEATURE_ZMM;
+		if (b & (1 << 30))
+			features |= X86_CPU_FEATURE_AVX512BW;
+		if (b & (1U << 31))
+			features |= X86_CPU_FEATURE_AVX512VL;
+		if (c & (1 << 11))
+			features |= X86_CPU_FEATURE_AVX512VNNI;
+	}
+
+	/* EAX=7, ECX=1: Extended Features */
+	cpuid(7, 1, &a, &b, &c, &d);
+	if ((a & (1 << 4)) && ((xcr0 & 0x6) == 0x6))
+		features |= X86_CPU_FEATURE_AVXVNNI;
+
+out:
+	disable_cpu_features_for_testing(&features, x86_cpu_feature_table,
+					 ARRAY_LEN(x86_cpu_feature_table));
+
+	libdeflate_x86_cpu_features = features | X86_CPU_FEATURES_KNOWN;
+}
+
+#endif /* X86_CPU_FEATURES_KNOWN */

data/ext/deflate_ruby/libdeflate/x86/cpu_features.h

@@ -0,0 +1,170 @@
+/*
+ * x86/cpu_features.h - feature detection for x86 CPUs
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_X86_CPU_FEATURES_H
+#define LIB_X86_CPU_FEATURES_H
+
+#include "../lib_common.h"
+
+#if defined(ARCH_X86_32) || defined(ARCH_X86_64)
+
+#define X86_CPU_FEATURE_SSE2		(1 << 0)
+#define X86_CPU_FEATURE_PCLMULQDQ	(1 << 1)
+#define X86_CPU_FEATURE_AVX		(1 << 2)
+#define X86_CPU_FEATURE_AVX2		(1 << 3)
+#define X86_CPU_FEATURE_BMI2		(1 << 4)
+/*
+ * ZMM indicates whether 512-bit vectors (zmm registers) should be used.  On
+ * some CPUs, to avoid downclocking issues we don't set ZMM even if the CPU and
+ * operating system support AVX-512.  On these CPUs, we may still use AVX-512
+ * instructions, but only with xmm and ymm registers.
+ */
+#define X86_CPU_FEATURE_ZMM		(1 << 5)
+#define X86_CPU_FEATURE_AVX512BW	(1 << 6)
+#define X86_CPU_FEATURE_AVX512VL	(1 << 7)
+#define X86_CPU_FEATURE_VPCLMULQDQ	(1 << 8)
+#define X86_CPU_FEATURE_AVX512VNNI	(1 << 9)
+#define X86_CPU_FEATURE_AVXVNNI		(1 << 10)
+
+#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
+/* Runtime x86 CPU feature detection is supported. */
+#  define X86_CPU_FEATURES_KNOWN	(1U << 31)
+extern volatile u32 libdeflate_x86_cpu_features;
+
+void libdeflate_init_x86_cpu_features(void);
+
+static inline u32 get_x86_cpu_features(void)
+{
+	if (libdeflate_x86_cpu_features == 0)
+		libdeflate_init_x86_cpu_features();
+	return libdeflate_x86_cpu_features;
+}
+/*
+ * x86 intrinsics are also supported.  Include the headers needed to use them.
+ * Normally just immintrin.h suffices.  With clang in MSVC compatibility mode,
+ * immintrin.h incorrectly skips including sub-headers, so include those too.
+ */
+#  include <immintrin.h>
+#  if defined(_MSC_VER) && defined(__clang__)
+#    include <tmmintrin.h>
+#    include <smmintrin.h>
+#    include <wmmintrin.h>
+#    include <avxintrin.h>
+#    include <avx2intrin.h>
+#    include <avx512fintrin.h>
+#    include <avx512bwintrin.h>
+#    include <avx512vlintrin.h>
+#    if __has_include(<avx512vlbwintrin.h>)
+#      include <avx512vlbwintrin.h>
+#    endif
+#    if __has_include(<vpclmulqdqintrin.h>)
+#      include <vpclmulqdqintrin.h>
+#    endif
+#    if __has_include(<avx512vnniintrin.h>)
+#      include <avx512vnniintrin.h>
+#    endif
+#    if __has_include(<avx512vlvnniintrin.h>)
+#      include <avx512vlvnniintrin.h>
+#    endif
+#    if __has_include(<avxvnniintrin.h>)
+#      include <avxvnniintrin.h>
+#    endif
+#  endif
+#else
+static inline u32 get_x86_cpu_features(void) { return 0; }
+#endif
+
+#if defined(__SSE2__) || \
+	(defined(_MSC_VER) && \
+	 (defined(ARCH_X86_64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)))
+#  define HAVE_SSE2(features)		1
+#  define HAVE_SSE2_NATIVE		1
+#else
+#  define HAVE_SSE2(features)		((features) & X86_CPU_FEATURE_SSE2)
+#  define HAVE_SSE2_NATIVE		0
+#endif
+
+#if (defined(__PCLMUL__) && defined(__SSE4_1__)) || \
+	(defined(_MSC_VER) && defined(__AVX2__))
+#  define HAVE_PCLMULQDQ(features)	1
+#else
+#  define HAVE_PCLMULQDQ(features)	((features) & X86_CPU_FEATURE_PCLMULQDQ)
+#endif
+
+#ifdef __AVX__
+#  define HAVE_AVX(features)		1
+#else
+#  define HAVE_AVX(features)		((features) & X86_CPU_FEATURE_AVX)
+#endif
+
+#ifdef __AVX2__
+#  define HAVE_AVX2(features)		1
+#else
+#  define HAVE_AVX2(features)		((features) & X86_CPU_FEATURE_AVX2)
+#endif
+
+#if defined(__BMI2__) || (defined(_MSC_VER) && defined(__AVX2__))
+#  define HAVE_BMI2(features)		1
+#  define HAVE_BMI2_NATIVE		1
+#else
+#  define HAVE_BMI2(features)		((features) & X86_CPU_FEATURE_BMI2)
+#  define HAVE_BMI2_NATIVE		0
+#endif
+
+#ifdef __AVX512BW__
+#  define HAVE_AVX512BW(features)	1
+#else
+#  define HAVE_AVX512BW(features)	((features) & X86_CPU_FEATURE_AVX512BW)
+#endif
+
+#ifdef __AVX512VL__
+#  define HAVE_AVX512VL(features)	1
+#else
+#  define HAVE_AVX512VL(features)	((features) & X86_CPU_FEATURE_AVX512VL)
+#endif
+
+#ifdef __VPCLMULQDQ__
+#  define HAVE_VPCLMULQDQ(features)	1
+#else
+#  define HAVE_VPCLMULQDQ(features)	((features) & X86_CPU_FEATURE_VPCLMULQDQ)
+#endif
+
+#ifdef __AVX512VNNI__
+#  define HAVE_AVX512VNNI(features)	1
+#else
+#  define HAVE_AVX512VNNI(features)	((features) & X86_CPU_FEATURE_AVX512VNNI)
+#endif
+
+#ifdef __AVXVNNI__
+#  define HAVE_AVXVNNI(features)	1
+#else
+#  define HAVE_AVXVNNI(features)	((features) & X86_CPU_FEATURE_AVXVNNI)
+#endif
+
+#endif /* ARCH_X86_32 || ARCH_X86_64 */
+
+#endif /* LIB_X86_CPU_FEATURES_H */

data/ext/deflate_ruby/libdeflate/x86/crc32_impl.h

@@ -0,0 +1,159 @@
+/*
+ * x86/crc32_impl.h - x86 implementations of the gzip CRC-32 algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_X86_CRC32_IMPL_H
+#define LIB_X86_CRC32_IMPL_H
+
+#include "cpu_features.h"
+
+/*
+ * pshufb(x, shift_tab[len..len+15]) left shifts x by 16-len bytes.
+ * pshufb(x, shift_tab[len+16..len+31]) right shifts x by len bytes.
+ */
+static const u8 MAYBE_UNUSED shift_tab[48] = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+};
+
+#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
+/*
+ * PCLMULQDQ implementation.  This targets PCLMULQDQ+SSE4.1, since in practice
+ * all CPUs that support PCLMULQDQ also support SSE4.1.
+ */
+#  define crc32_x86_pclmulqdq	crc32_x86_pclmulqdq
+#  define SUFFIX			 _pclmulqdq
+#  define ATTRIBUTES		_target_attribute("pclmul,sse4.1")
+#  define VL			16
+#  define USE_AVX512		0
+#  include "crc32_pclmul_template.h"
+
+/*
+ * PCLMULQDQ/AVX implementation.  Same as above, but this is compiled with AVX
+ * enabled so that the compiler can generate VEX-coded instructions which can be
+ * slightly more efficient.  It still uses 128-bit vectors.
+ */
+#  define crc32_x86_pclmulqdq_avx	crc32_x86_pclmulqdq_avx
+#  define SUFFIX				 _pclmulqdq_avx
+#  define ATTRIBUTES		_target_attribute("pclmul,avx")
+#  define VL			16
+#  define USE_AVX512		0
+#  include "crc32_pclmul_template.h"
+#endif
+
+/*
+ * VPCLMULQDQ/AVX2 implementation.  This is used on CPUs that have AVX2 and
+ * VPCLMULQDQ but don't have AVX-512, for example Intel Alder Lake.
+ *
+ * Currently this can't be enabled with MSVC because MSVC has a bug where it
+ * incorrectly assumes that VPCLMULQDQ implies AVX-512:
+ * https://developercommunity.visualstudio.com/t/Compiler-incorrectly-assumes-VAES-and-VP/10578785
+ *
+ * gcc 8.1 and 8.2 had a similar bug where they assumed that
+ * _mm256_clmulepi64_epi128() always needed AVX512.  It's fixed in gcc 8.3.
+ *
+ * _mm256_zextsi128_si256() requires gcc 10.
+ */
+#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
+#  define crc32_x86_vpclmulqdq_avx2	crc32_x86_vpclmulqdq_avx2
+#  define SUFFIX				 _vpclmulqdq_avx2
+#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx2")
+#  define VL			32
+#  define USE_AVX512		0
+#  include "crc32_pclmul_template.h"
+#endif
+
+#if (GCC_PREREQ(10, 1) || CLANG_PREREQ(6, 0, 10000000) || MSVC_PREREQ(1920)) && \
+	!defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_VPCLMULQDQ)
+/*
+ * VPCLMULQDQ/AVX512 implementation using 256-bit vectors.  This is very similar
+ * to the VPCLMULQDQ/AVX2 implementation but takes advantage of the vpternlog
+ * instruction and more registers.  This is used on certain older Intel CPUs,
+ * specifically Ice Lake and Tiger Lake, which support VPCLMULQDQ and AVX512 but
+ * downclock a bit too eagerly when ZMM registers are used.
+ *
+ * _mm256_zextsi128_si256() requires gcc 10.
+ */
+#  define crc32_x86_vpclmulqdq_avx512_vl256  crc32_x86_vpclmulqdq_avx512_vl256
+#  define SUFFIX				      _vpclmulqdq_avx512_vl256
+#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
+#  define VL			32
+#  define USE_AVX512		1
+#  include "crc32_pclmul_template.h"
+
+/*
+ * VPCLMULQDQ/AVX512 implementation using 512-bit vectors.  This is used on CPUs
+ * that have a good AVX-512 implementation including VPCLMULQDQ.
+ *
+ * _mm512_zextsi128_si512() requires gcc 10.
+ */
+#  define crc32_x86_vpclmulqdq_avx512_vl512  crc32_x86_vpclmulqdq_avx512_vl512
+#  define SUFFIX				      _vpclmulqdq_avx512_vl512
+#  define ATTRIBUTES		_target_attribute("vpclmulqdq,pclmul,avx512bw,avx512vl")
+#  define VL			64
+#  define USE_AVX512		1
+#  include "crc32_pclmul_template.h"
+#endif
+
+static inline crc32_func_t
+arch_select_crc32_func(void)
+{
+	const u32 features MAYBE_UNUSED = get_x86_cpu_features();
+
+#ifdef crc32_x86_vpclmulqdq_avx512_vl512
+	if ((features & X86_CPU_FEATURE_ZMM) &&
+	    HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
+	    HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
+		return crc32_x86_vpclmulqdq_avx512_vl512;
+#endif
+#ifdef crc32_x86_vpclmulqdq_avx512_vl256
+	if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
+	    HAVE_AVX512BW(features) && HAVE_AVX512VL(features))
+		return crc32_x86_vpclmulqdq_avx512_vl256;
+#endif
+#ifdef crc32_x86_vpclmulqdq_avx2
+	if (HAVE_VPCLMULQDQ(features) && HAVE_PCLMULQDQ(features) &&
+	    HAVE_AVX2(features))
+		return crc32_x86_vpclmulqdq_avx2;
+#endif
+#ifdef crc32_x86_pclmulqdq_avx
+	if (HAVE_PCLMULQDQ(features) && HAVE_AVX(features))
+		return crc32_x86_pclmulqdq_avx;
+#endif
+#ifdef crc32_x86_pclmulqdq
+	if (HAVE_PCLMULQDQ(features))
+		return crc32_x86_pclmulqdq;
+#endif
+	return NULL;
+}
+#define arch_select_crc32_func	arch_select_crc32_func
+
+#endif /* LIB_X86_CRC32_IMPL_H */