deflate-ruby 0.1.0

data/ext/deflate_ruby/libdeflate/lib/arm/cpu_features.c

@@ -0,0 +1,230 @@
+/*
+ * arm/cpu_features.c - feature detection for ARM CPUs
+ *
+ * Copyright 2018 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * ARM CPUs don't have a standard way for unprivileged programs to detect CPU
+ * features.  But an OS-specific way can be used when available.
+ */
+
+#ifdef __APPLE__
+#  undef _ANSI_SOURCE
+#  undef _DARWIN_C_SOURCE
+#  define _DARWIN_C_SOURCE /* for sysctlbyname() */
+#endif
+
+#include "../cpu_features_common.h" /* must be included first */
+#include "cpu_features.h"
+
+#ifdef ARM_CPU_FEATURES_KNOWN
+/* Runtime ARM CPU feature detection is supported. */
+
+#ifdef __linux__
+/*
+ * On Linux, arm32 and arm64 CPU features can be detected by reading the
+ * AT_HWCAP and AT_HWCAP2 values from /proc/self/auxv.
+ *
+ * Ideally we'd use the C library function getauxval(), but it's not guaranteed
+ * to be available: it was only added to glibc in 2.16, and in Android it was
+ * added to API level 18 for arm32 and level 21 for arm64.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+
+#define AT_HWCAP	16
+#define AT_HWCAP2	26
+
+static void scan_auxv(unsigned long *hwcap, unsigned long *hwcap2)
+{
+	int fd;
+	unsigned long auxbuf[32];
+	int filled = 0;
+	int i;
+
+	fd = open("/proc/self/auxv", O_RDONLY);
+	if (fd < 0)
+		return;
+
+	for (;;) {
+		do {
+			int ret = read(fd, &((char *)auxbuf)[filled],
+				       sizeof(auxbuf) - filled);
+			if (ret <= 0) {
+				if (ret < 0 && errno == EINTR)
+					continue;
+				goto out;
+			}
+			filled += ret;
+		} while (filled < 2 * sizeof(long));
+
+		i = 0;
+		do {
+			unsigned long type = auxbuf[i];
+			unsigned long value = auxbuf[i + 1];
+
+			if (type == AT_HWCAP)
+				*hwcap = value;
+			else if (type == AT_HWCAP2)
+				*hwcap2 = value;
+			i += 2;
+			filled -= 2 * sizeof(long);
+		} while (filled >= 2 * sizeof(long));
+
+		memmove(auxbuf, &auxbuf[i], filled);
+	}
+out:
+	close(fd);
+}
+
+static u32 query_arm_cpu_features(void)
+{
+	u32 features = 0;
+	unsigned long hwcap = 0;
+	unsigned long hwcap2 = 0;
+
+	scan_auxv(&hwcap, &hwcap2);
+
+#ifdef ARCH_ARM32
+	STATIC_ASSERT(sizeof(long) == 4);
+	if (hwcap & (1 << 12))	/* HWCAP_NEON */
+		features |= ARM_CPU_FEATURE_NEON;
+#else
+	STATIC_ASSERT(sizeof(long) == 8);
+	if (hwcap & (1 << 1))	/* HWCAP_ASIMD */
+		features |= ARM_CPU_FEATURE_NEON;
+	if (hwcap & (1 << 4))	/* HWCAP_PMULL */
+		features |= ARM_CPU_FEATURE_PMULL;
+	if (hwcap & (1 << 7))	/* HWCAP_CRC32 */
+		features |= ARM_CPU_FEATURE_CRC32;
+	if (hwcap & (1 << 17))	/* HWCAP_SHA3 */
+		features |= ARM_CPU_FEATURE_SHA3;
+	if (hwcap & (1 << 20))	/* HWCAP_ASIMDDP */
+		features |= ARM_CPU_FEATURE_DOTPROD;
+#endif
+	return features;
+}
+
+#elif defined(__APPLE__)
+/* On Apple platforms, arm64 CPU features can be detected via sysctlbyname(). */
+
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <TargetConditionals.h>
+
+static const struct {
+	const char *name;
+	u32 feature;
+} feature_sysctls[] = {
+	{ "hw.optional.neon",		  ARM_CPU_FEATURE_NEON },
+	{ "hw.optional.AdvSIMD",	  ARM_CPU_FEATURE_NEON },
+	{ "hw.optional.arm.FEAT_PMULL",	  ARM_CPU_FEATURE_PMULL },
+	{ "hw.optional.armv8_crc32",	  ARM_CPU_FEATURE_CRC32 },
+	{ "hw.optional.armv8_2_sha3",	  ARM_CPU_FEATURE_SHA3 },
+	{ "hw.optional.arm.FEAT_SHA3",	  ARM_CPU_FEATURE_SHA3 },
+	{ "hw.optional.arm.FEAT_DotProd", ARM_CPU_FEATURE_DOTPROD },
+};
+
+static u32 query_arm_cpu_features(void)
+{
+	u32 features = 0;
+	size_t i;
+
+	for (i = 0; i < ARRAY_LEN(feature_sysctls); i++) {
+		const char *name = feature_sysctls[i].name;
+		u32 val = 0;
+		size_t valsize = sizeof(val);
+
+		if (sysctlbyname(name, &val, &valsize, NULL, 0) == 0 &&
+		    valsize == sizeof(val) && val == 1)
+			features |= feature_sysctls[i].feature;
+	}
+	return features;
+}
+#elif defined(_WIN32)
+
+#include <windows.h>
+
+#ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE /* added in Windows SDK 20348 */
+#  define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
+#endif
+
+static u32 query_arm_cpu_features(void)
+{
+	u32 features = ARM_CPU_FEATURE_NEON;
+
+	if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE))
+		features |= ARM_CPU_FEATURE_PMULL;
+	if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE))
+		features |= ARM_CPU_FEATURE_CRC32;
+	if (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE))
+		features |= ARM_CPU_FEATURE_DOTPROD;
+
+	/* FIXME: detect SHA3 support too. */
+
+	return features;
+}
+#else
+#error "unhandled case"
+#endif
+
+static const struct cpu_feature arm_cpu_feature_table[] = {
+	{ARM_CPU_FEATURE_NEON,		"neon"},
+	{ARM_CPU_FEATURE_PMULL,		"pmull"},
+	{ARM_CPU_FEATURE_PREFER_PMULL,  "prefer_pmull"},
+	{ARM_CPU_FEATURE_CRC32,		"crc32"},
+	{ARM_CPU_FEATURE_SHA3,		"sha3"},
+	{ARM_CPU_FEATURE_DOTPROD,	"dotprod"},
+};
+
+volatile u32 libdeflate_arm_cpu_features = 0;
+
+void libdeflate_init_arm_cpu_features(void)
+{
+	u32 features = query_arm_cpu_features();
+
+	/*
+	 * On the Apple M1 processor, crc32 instructions max out at about 25.5
+	 * GB/s in the best case of using a 3-way or greater interleaved chunked
+	 * implementation, whereas a pmull-based implementation achieves 68 GB/s
+	 * provided that the stride length is large enough (about 10+ vectors
+	 * with eor3, or 12+ without).
+	 *
+	 * Assume that crc32 instructions are preferable in other cases.
+	 */
+#if (defined(__APPLE__) && TARGET_OS_OSX) || defined(TEST_SUPPORT__DO_NOT_USE)
+	features |= ARM_CPU_FEATURE_PREFER_PMULL;
+#endif
+
+	disable_cpu_features_for_testing(&features, arm_cpu_feature_table,
+					 ARRAY_LEN(arm_cpu_feature_table));
+
+	libdeflate_arm_cpu_features = features | ARM_CPU_FEATURES_KNOWN;
+}
+
+#endif /* ARM_CPU_FEATURES_KNOWN */

data/ext/deflate_ruby/libdeflate/lib/arm/cpu_features.h

@@ -0,0 +1,214 @@
+/*
+ * arm/cpu_features.h - feature detection for ARM CPUs
+ *
+ * Copyright 2018 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_CPU_FEATURES_H
+#define LIB_ARM_CPU_FEATURES_H
+
+#include "../lib_common.h"
+
+#if defined(ARCH_ARM32) || defined(ARCH_ARM64)
+
+#define ARM_CPU_FEATURE_NEON		(1 << 0)
+#define ARM_CPU_FEATURE_PMULL		(1 << 1)
+/*
+ * PREFER_PMULL indicates that the CPU has very high pmull throughput, and so
+ * the 12x wide pmull-based CRC-32 implementation is likely to be faster than an
+ * implementation based on the crc32 instructions.
+ */
+#define ARM_CPU_FEATURE_PREFER_PMULL	(1 << 2)
+#define ARM_CPU_FEATURE_CRC32		(1 << 3)
+#define ARM_CPU_FEATURE_SHA3		(1 << 4)
+#define ARM_CPU_FEATURE_DOTPROD		(1 << 5)
+
+#if !defined(FREESTANDING) && \
+    (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
+    (defined(__linux__) || \
+     (defined(__APPLE__) && defined(ARCH_ARM64)) || \
+     (defined(_WIN32) && defined(ARCH_ARM64)))
+/* Runtime ARM CPU feature detection is supported. */
+#  define ARM_CPU_FEATURES_KNOWN	(1U << 31)
+extern volatile u32 libdeflate_arm_cpu_features;
+
+void libdeflate_init_arm_cpu_features(void);
+
+static inline u32 get_arm_cpu_features(void)
+{
+	if (libdeflate_arm_cpu_features == 0)
+		libdeflate_init_arm_cpu_features();
+	return libdeflate_arm_cpu_features;
+}
+#else
+static inline u32 get_arm_cpu_features(void) { return 0; }
+#endif
+
+/* NEON */
+#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))
+#  define HAVE_NEON(features)	1
+#  define HAVE_NEON_NATIVE	1
+#else
+#  define HAVE_NEON(features)	((features) & ARM_CPU_FEATURE_NEON)
+#  define HAVE_NEON_NATIVE	0
+#endif
+/*
+ * With both gcc and clang, NEON intrinsics require that the main target has
+ * NEON enabled already.  Exception: with gcc 6.1 and later (r230411 for arm32,
+ * r226563 for arm64), hardware floating point support is sufficient.
+ */
+#if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
+	(HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))
+#  define HAVE_NEON_INTRIN	1
+#  include <arm_neon.h>
+#else
+#  define HAVE_NEON_INTRIN	0
+#endif
+
+/* PMULL */
+#ifdef __ARM_FEATURE_CRYPTO
+#  define HAVE_PMULL(features)	1
+#else
+#  define HAVE_PMULL(features)	((features) & ARM_CPU_FEATURE_PMULL)
+#endif
+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
+	(GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
+	CPU_IS_LITTLE_ENDIAN() /* untested on big endian */
+#  define HAVE_PMULL_INTRIN	1
+   /* Work around MSVC's vmull_p64() taking poly64x1_t instead of poly64_t */
+#  ifdef _MSC_VER
+#    define compat_vmull_p64(a, b)  vmull_p64(vcreate_p64(a), vcreate_p64(b))
+#  else
+#    define compat_vmull_p64(a, b)  vmull_p64((a), (b))
+#  endif
+#else
+#  define HAVE_PMULL_INTRIN	0
+#endif
+
+/* CRC32 */
+#ifdef __ARM_FEATURE_CRC32
+#  define HAVE_CRC32(features)	1
+#else
+#  define HAVE_CRC32(features)	((features) & ARM_CPU_FEATURE_CRC32)
+#endif
+#if defined(ARCH_ARM64) && \
+	(defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
+#  define HAVE_CRC32_INTRIN	1
+#  if defined(__GNUC__) || defined(__clang__)
+#    include <arm_acle.h>
+#  endif
+   /*
+    * Use an inline assembly fallback for clang 15 and earlier, which only
+    * defined the crc32 intrinsics when crc32 is enabled in the main target.
+    */
+#  if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
+	!defined(__ARM_FEATURE_CRC32)
+#    undef __crc32b
+#    define __crc32b(a, b)					\
+	({ uint32_t res;					\
+	   __asm__("crc32b %w0, %w1, %w2"			\
+		   : "=r" (res) : "r" (a), "r" (b));		\
+	   res; })
+#    undef __crc32h
+#    define __crc32h(a, b)					\
+	({ uint32_t res;					\
+	   __asm__("crc32h %w0, %w1, %w2"			\
+		   : "=r" (res) : "r" (a), "r" (b));		\
+	   res; })
+#    undef __crc32w
+#    define __crc32w(a, b)					\
+	({ uint32_t res;					\
+	   __asm__("crc32w %w0, %w1, %w2"			\
+		   : "=r" (res) : "r" (a), "r" (b));		\
+	   res; })
+#    undef __crc32d
+#    define __crc32d(a, b)					\
+	({ uint32_t res;					\
+	   __asm__("crc32x %w0, %w1, %2"			\
+		   : "=r" (res) : "r" (a), "r" (b));		\
+	   res; })
+#    pragma clang diagnostic ignored "-Wgnu-statement-expression"
+#  endif
+#else
+#  define HAVE_CRC32_INTRIN	0
+#endif
+
+/* SHA3 (needed for the eor3 instruction) */
+#ifdef __ARM_FEATURE_SHA3
+#  define HAVE_SHA3(features)	1
+#else
+#  define HAVE_SHA3(features)	((features) & ARM_CPU_FEATURE_SHA3)
+#endif
+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
+	(GCC_PREREQ(9, 1) /* r268049 */ || \
+	 CLANG_PREREQ(7, 0, 10010463) /* r338010 */)
+#  define HAVE_SHA3_INTRIN	1
+   /*
+    * Use an inline assembly fallback for clang 15 and earlier, which only
+    * defined the sha3 intrinsics when sha3 is enabled in the main target.
+    */
+#  if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
+	!defined(__ARM_FEATURE_SHA3)
+#    undef veor3q_u8
+#    define veor3q_u8(a, b, c)					\
+	({ uint8x16_t res;					\
+	   __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b"	\
+		   : "=w" (res) : "w" (a), "w" (b), "w" (c));	\
+	   res; })
+#    pragma clang diagnostic ignored "-Wgnu-statement-expression"
+#  endif
+#else
+#  define HAVE_SHA3_INTRIN	0
+#endif
+
+/* dotprod */
+#ifdef __ARM_FEATURE_DOTPROD
+#  define HAVE_DOTPROD(features)	1
+#else
+#  define HAVE_DOTPROD(features)	((features) & ARM_CPU_FEATURE_DOTPROD)
+#endif
+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
+	(GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))
+#  define HAVE_DOTPROD_INTRIN	1
+   /*
+    * Use an inline assembly fallback for clang 15 and earlier, which only
+    * defined the dotprod intrinsics when dotprod is enabled in the main target.
+    */
+#  if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
+	!defined(__ARM_FEATURE_DOTPROD)
+#    undef vdotq_u32
+#    define vdotq_u32(a, b, c)					\
+	({ uint32x4_t res = (a);				\
+	   __asm__("udot %0.4s, %1.16b, %2.16b"			\
+		   : "+w" (res) : "w" (b), "w" (c));		\
+	   res; })
+#    pragma clang diagnostic ignored "-Wgnu-statement-expression"
+#  endif
+#else
+#  define HAVE_DOTPROD_INTRIN	0
+#endif
+
+#endif /* ARCH_ARM32 || ARCH_ARM64 */
+
+#endif /* LIB_ARM_CPU_FEATURES_H */