deflate-ruby 0.1.0

data/ext/deflate_ruby/libdeflate/lib/adler32.c

@@ -0,0 +1,162 @@
+/*
+ * adler32.c - Adler-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "lib_common.h"
+
+/* The Adler-32 divisor, or "base", value */
+#define DIVISOR 65521
+
+/*
+ * MAX_CHUNK_LEN is the most bytes that can be processed without the possibility
+ * of s2 overflowing when it is represented as an unsigned 32-bit integer.  This
+ * value was computed using the following Python script:
+ *
+ *	divisor = 65521
+ *	count = 0
+ *	s1 = divisor - 1
+ *	s2 = divisor - 1
+ *	while True:
+ *		s1 += 0xFF
+ *		s2 += s1
+ *		if s2 > 0xFFFFFFFF:
+ *			break
+ *		count += 1
+ *	print(count)
+ *
+ * Note that to get the correct worst-case value, we must assume that every byte
+ * has value 0xFF and that s1 and s2 started with the highest possible values
+ * modulo the divisor.
+ */
+#define MAX_CHUNK_LEN	5552
+
+/*
+ * Update the Adler-32 values s1 and s2 using n bytes from p, update p to p + n,
+ * update n to 0, and reduce s1 and s2 mod DIVISOR.  It is assumed that neither
+ * s1 nor s2 can overflow before the reduction at the end, i.e. n plus any bytes
+ * already processed after the last reduction must not exceed MAX_CHUNK_LEN.
+ *
+ * This uses only portable C code.  This is used as a fallback when a vectorized
+ * implementation of Adler-32 (e.g. AVX2) is unavailable on the platform.
+ *
+ * Some of the vectorized implementations also use this to handle the end of the
+ * data when the data isn't evenly divisible by the length the vectorized code
+ * works on.  To avoid compiler errors about target-specific option mismatches
+ * when this is used in that way, this is a macro rather than a function.
+ *
+ * Although this is unvectorized, this does include an optimization where the
+ * main loop processes four bytes at a time using a strategy similar to that
+ * used by vectorized implementations.  This provides increased instruction-
+ * level parallelism compared to the traditional 's1 += *p++; s2 += s1;'.
+ */
+#define ADLER32_CHUNK(s1, s2, p, n)					\
+do {									\
+	if (n >= 4) {							\
+		u32 s1_sum = 0;						\
+		u32 byte_0_sum = 0;					\
+		u32 byte_1_sum = 0;					\
+		u32 byte_2_sum = 0;					\
+		u32 byte_3_sum = 0;					\
+									\
+		do {							\
+			s1_sum += s1;					\
+			s1 += p[0] + p[1] + p[2] + p[3];		\
+			byte_0_sum += p[0];				\
+			byte_1_sum += p[1];				\
+			byte_2_sum += p[2];				\
+			byte_3_sum += p[3];				\
+			p += 4;						\
+			n -= 4;						\
+		} while (n >= 4);					\
+		s2 += (4 * (s1_sum + byte_0_sum)) + (3 * byte_1_sum) +	\
+		      (2 * byte_2_sum) + byte_3_sum;			\
+	}								\
+	for (; n; n--, p++) {						\
+		s1 += *p;						\
+		s2 += s1;						\
+	}								\
+	s1 %= DIVISOR;							\
+	s2 %= DIVISOR;							\
+} while (0)
+
+static u32 MAYBE_UNUSED
+adler32_generic(u32 adler, const u8 *p, size_t len)
+{
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
+
+	while (len) {
+		size_t n = MIN(len, MAX_CHUNK_LEN & ~3);
+
+		len -= n;
+		ADLER32_CHUNK(s1, s2, p, n);
+	}
+
+	return (s2 << 16) | s1;
+}
+
+/* Include architecture-specific implementation(s) if available. */
+#undef DEFAULT_IMPL
+#undef arch_select_adler32_func
+typedef u32 (*adler32_func_t)(u32 adler, const u8 *p, size_t len);
+#if defined(ARCH_ARM32) || defined(ARCH_ARM64)
+#  include "arm/adler32_impl.h"
+#elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
+#  include "x86/adler32_impl.h"
+#endif
+
+#ifndef DEFAULT_IMPL
+#  define DEFAULT_IMPL adler32_generic
+#endif
+
+#ifdef arch_select_adler32_func
+static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len);
+
+static volatile adler32_func_t adler32_impl = dispatch_adler32;
+
+/* Choose the best implementation at runtime. */
+static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len)
+{
+	adler32_func_t f = arch_select_adler32_func();
+
+	if (f == NULL)
+		f = DEFAULT_IMPL;
+
+	adler32_impl = f;
+	return f(adler, p, len);
+}
+#else
+/* The best implementation is statically known, so call it directly. */
+#define adler32_impl DEFAULT_IMPL
+#endif
+
+LIBDEFLATEAPI u32
+libdeflate_adler32(u32 adler, const void *buffer, size_t len)
+{
+	if (buffer == NULL) /* Return initial value. */
+		return 1;
+	return adler32_impl(adler, buffer, len);
+}

data/ext/deflate_ruby/libdeflate/lib/arm/adler32_impl.h

@@ -0,0 +1,358 @@
+/*
+ * arm/adler32_impl.h - ARM implementations of Adler-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_ADLER32_IMPL_H
+#define LIB_ARM_ADLER32_IMPL_H
+
+#include "cpu_features.h"
+
+/* Regular NEON implementation */
+#if HAVE_NEON_INTRIN && CPU_IS_LITTLE_ENDIAN()
+#  define adler32_arm_neon	adler32_arm_neon
+#  if HAVE_NEON_NATIVE
+     /*
+      * Use no attributes if none are needed, to support old versions of clang
+      * that don't accept the simd target attribute.
+      */
+#    define ATTRIBUTES
+#  elif defined(ARCH_ARM32)
+#    define ATTRIBUTES	_target_attribute("fpu=neon")
+#  elif defined(__clang__)
+#    define ATTRIBUTES	_target_attribute("simd")
+#  else
+#    define ATTRIBUTES	_target_attribute("+simd")
+#  endif
+static ATTRIBUTES MAYBE_UNUSED u32
+adler32_arm_neon(u32 adler, const u8 *p, size_t len)
+{
+	static const u16 _aligned_attribute(16) mults[64] = {
+		64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+		48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const uint16x8_t mults_a = vld1q_u16(&mults[0]);
+	const uint16x8_t mults_b = vld1q_u16(&mults[8]);
+	const uint16x8_t mults_c = vld1q_u16(&mults[16]);
+	const uint16x8_t mults_d = vld1q_u16(&mults[24]);
+	const uint16x8_t mults_e = vld1q_u16(&mults[32]);
+	const uint16x8_t mults_f = vld1q_u16(&mults[40]);
+	const uint16x8_t mults_g = vld1q_u16(&mults[48]);
+	const uint16x8_t mults_h = vld1q_u16(&mults[56]);
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
+
+	/*
+	 * If the length is large and the pointer is misaligned, align it.
+	 * For smaller lengths, just take the misaligned load penalty.
+	 */
+	if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {
+		do {
+			s1 += *p++;
+			s2 += s1;
+			len--;
+		} while ((uintptr_t)p & 15);
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+	while (len) {
+		/*
+		 * Calculate the length of the next data chunk such that s1 and
+		 * s2 are guaranteed to not exceed UINT32_MAX.
+		 */
+		size_t n = MIN(len, MAX_CHUNK_LEN & ~63);
+
+		len -= n;
+
+		if (n >= 64) {
+			uint32x4_t v_s1 = vdupq_n_u32(0);
+			uint32x4_t v_s2 = vdupq_n_u32(0);
+			/*
+			 * v_byte_sums_* contain the sum of the bytes at index i
+			 * across all 64-byte segments, for each index 0..63.
+			 */
+			uint16x8_t v_byte_sums_a = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_b = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_c = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_d = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_e = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_f = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_g = vdupq_n_u16(0);
+			uint16x8_t v_byte_sums_h = vdupq_n_u16(0);
+
+			s2 += s1 * (n & ~63);
+
+			do {
+				/* Load the next 64 data bytes. */
+				const uint8x16_t data_a = vld1q_u8(p + 0);
+				const uint8x16_t data_b = vld1q_u8(p + 16);
+				const uint8x16_t data_c = vld1q_u8(p + 32);
+				const uint8x16_t data_d = vld1q_u8(p + 48);
+				uint16x8_t tmp;
+
+				/*
+				 * Accumulate the previous s1 counters into the
+				 * s2 counters.  The needed multiplication by 64
+				 * is delayed to later.
+				 */
+				v_s2 = vaddq_u32(v_s2, v_s1);
+
+				/*
+				 * Add the 64 data bytes to their v_byte_sums
+				 * counters, while also accumulating the sums of
+				 * each adjacent set of 4 bytes into v_s1.
+				 */
+				tmp = vpaddlq_u8(data_a);
+				v_byte_sums_a = vaddw_u8(v_byte_sums_a,
+							 vget_low_u8(data_a));
+				v_byte_sums_b = vaddw_u8(v_byte_sums_b,
+							 vget_high_u8(data_a));
+				tmp = vpadalq_u8(tmp, data_b);
+				v_byte_sums_c = vaddw_u8(v_byte_sums_c,
+							 vget_low_u8(data_b));
+				v_byte_sums_d = vaddw_u8(v_byte_sums_d,
+							 vget_high_u8(data_b));
+				tmp = vpadalq_u8(tmp, data_c);
+				v_byte_sums_e = vaddw_u8(v_byte_sums_e,
+							 vget_low_u8(data_c));
+				v_byte_sums_f = vaddw_u8(v_byte_sums_f,
+							 vget_high_u8(data_c));
+				tmp = vpadalq_u8(tmp, data_d);
+				v_byte_sums_g = vaddw_u8(v_byte_sums_g,
+							 vget_low_u8(data_d));
+				v_byte_sums_h = vaddw_u8(v_byte_sums_h,
+							 vget_high_u8(data_d));
+				v_s1 = vpadalq_u16(v_s1, tmp);
+
+				p += 64;
+				n -= 64;
+			} while (n >= 64);
+
+			/* s2 = 64*s2 + (64*bytesum0 + 63*bytesum1 + ... + 1*bytesum63) */
+		#ifdef ARCH_ARM32
+		#  define umlal2(a, b, c)  vmlal_u16((a), vget_high_u16(b), vget_high_u16(c))
+		#else
+		#  define umlal2	   vmlal_high_u16
+		#endif
+			v_s2 = vqshlq_n_u32(v_s2, 6);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_a),
+					 vget_low_u16(mults_a));
+			v_s2 = umlal2(v_s2, v_byte_sums_a, mults_a);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_b),
+					 vget_low_u16(mults_b));
+			v_s2 = umlal2(v_s2, v_byte_sums_b, mults_b);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_c),
+					 vget_low_u16(mults_c));
+			v_s2 = umlal2(v_s2, v_byte_sums_c, mults_c);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_d),
+					 vget_low_u16(mults_d));
+			v_s2 = umlal2(v_s2, v_byte_sums_d, mults_d);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_e),
+					 vget_low_u16(mults_e));
+			v_s2 = umlal2(v_s2, v_byte_sums_e, mults_e);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_f),
+					 vget_low_u16(mults_f));
+			v_s2 = umlal2(v_s2, v_byte_sums_f, mults_f);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_g),
+					 vget_low_u16(mults_g));
+			v_s2 = umlal2(v_s2, v_byte_sums_g, mults_g);
+			v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_h),
+					 vget_low_u16(mults_h));
+			v_s2 = umlal2(v_s2, v_byte_sums_h, mults_h);
+		#undef umlal2
+
+			/* Horizontal sum to finish up */
+		#ifdef ARCH_ARM32
+			s1 += vgetq_lane_u32(v_s1, 0) + vgetq_lane_u32(v_s1, 1) +
+			      vgetq_lane_u32(v_s1, 2) + vgetq_lane_u32(v_s1, 3);
+			s2 += vgetq_lane_u32(v_s2, 0) + vgetq_lane_u32(v_s2, 1) +
+			      vgetq_lane_u32(v_s2, 2) + vgetq_lane_u32(v_s2, 3);
+		#else
+			s1 += vaddvq_u32(v_s1);
+			s2 += vaddvq_u32(v_s2);
+		#endif
+		}
+		/*
+		 * Process the last 0 <= n < 64 bytes of the chunk using
+		 * scalar instructions and reduce s1 and s2 mod DIVISOR.
+		 */
+		ADLER32_CHUNK(s1, s2, p, n);
+	}
+	return (s2 << 16) | s1;
+}
+#undef ATTRIBUTES
+#endif /* Regular NEON implementation */
+
+/* NEON+dotprod implementation */
+#if HAVE_DOTPROD_INTRIN && CPU_IS_LITTLE_ENDIAN()
+#  define adler32_arm_neon_dotprod	adler32_arm_neon_dotprod
+#  ifdef __clang__
+#    define ATTRIBUTES	_target_attribute("dotprod")
+   /*
+    * With gcc 13.1 and earlier (before gcc commit 73d3bc348190 or 9aac37ab8a7b,
+    * "aarch64: Remove architecture dependencies from intrinsics"),
+    * arch=armv8.2-a is needed for the dotprod intrinsics, unless the default
+    * target is armv8.3-a or later in which case it must be omitted.  armv8.3-a
+    * or later can be detected by checking for __ARM_FEATURE_JCVT.
+    */
+#  elif GCC_PREREQ(13, 2) || defined(__ARM_FEATURE_JCVT)
+#    define ATTRIBUTES	_target_attribute("+dotprod")
+#  else
+#    define ATTRIBUTES	_target_attribute("arch=armv8.2-a+dotprod")
+#  endif
+static ATTRIBUTES u32
+adler32_arm_neon_dotprod(u32 adler, const u8 *p, size_t len)
+{
+	static const u8 _aligned_attribute(16) mults[64] = {
+		64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+		48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+		32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+		16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,
+	};
+	const uint8x16_t mults_a = vld1q_u8(&mults[0]);
+	const uint8x16_t mults_b = vld1q_u8(&mults[16]);
+	const uint8x16_t mults_c = vld1q_u8(&mults[32]);
+	const uint8x16_t mults_d = vld1q_u8(&mults[48]);
+	const uint8x16_t ones = vdupq_n_u8(1);
+	u32 s1 = adler & 0xFFFF;
+	u32 s2 = adler >> 16;
+
+	/*
+	 * If the length is large and the pointer is misaligned, align it.
+	 * For smaller lengths, just take the misaligned load penalty.
+	 */
+	if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {
+		do {
+			s1 += *p++;
+			s2 += s1;
+			len--;
+		} while ((uintptr_t)p & 15);
+		s1 %= DIVISOR;
+		s2 %= DIVISOR;
+	}
+
+	while (len) {
+		/*
+		 * Calculate the length of the next data chunk such that s1 and
+		 * s2 are guaranteed to not exceed UINT32_MAX.
+		 */
+		size_t n = MIN(len, MAX_CHUNK_LEN & ~63);
+
+		len -= n;
+
+		if (n >= 64) {
+			uint32x4_t v_s1_a = vdupq_n_u32(0);
+			uint32x4_t v_s1_b = vdupq_n_u32(0);
+			uint32x4_t v_s1_c = vdupq_n_u32(0);
+			uint32x4_t v_s1_d = vdupq_n_u32(0);
+			uint32x4_t v_s2_a = vdupq_n_u32(0);
+			uint32x4_t v_s2_b = vdupq_n_u32(0);
+			uint32x4_t v_s2_c = vdupq_n_u32(0);
+			uint32x4_t v_s2_d = vdupq_n_u32(0);
+			uint32x4_t v_s1_sums_a = vdupq_n_u32(0);
+			uint32x4_t v_s1_sums_b = vdupq_n_u32(0);
+			uint32x4_t v_s1_sums_c = vdupq_n_u32(0);
+			uint32x4_t v_s1_sums_d = vdupq_n_u32(0);
+			uint32x4_t v_s1;
+			uint32x4_t v_s2;
+			uint32x4_t v_s1_sums;
+
+			s2 += s1 * (n & ~63);
+
+			do {
+				uint8x16_t data_a = vld1q_u8(p + 0);
+				uint8x16_t data_b = vld1q_u8(p + 16);
+				uint8x16_t data_c = vld1q_u8(p + 32);
+				uint8x16_t data_d = vld1q_u8(p + 48);
+
+				v_s1_sums_a = vaddq_u32(v_s1_sums_a, v_s1_a);
+				v_s1_a = vdotq_u32(v_s1_a, data_a, ones);
+				v_s2_a = vdotq_u32(v_s2_a, data_a, mults_a);
+
+				v_s1_sums_b = vaddq_u32(v_s1_sums_b, v_s1_b);
+				v_s1_b = vdotq_u32(v_s1_b, data_b, ones);
+				v_s2_b = vdotq_u32(v_s2_b, data_b, mults_b);
+
+				v_s1_sums_c = vaddq_u32(v_s1_sums_c, v_s1_c);
+				v_s1_c = vdotq_u32(v_s1_c, data_c, ones);
+				v_s2_c = vdotq_u32(v_s2_c, data_c, mults_c);
+
+				v_s1_sums_d = vaddq_u32(v_s1_sums_d, v_s1_d);
+				v_s1_d = vdotq_u32(v_s1_d, data_d, ones);
+				v_s2_d = vdotq_u32(v_s2_d, data_d, mults_d);
+
+				p += 64;
+				n -= 64;
+			} while (n >= 64);
+
+			v_s1 = vaddq_u32(vaddq_u32(v_s1_a, v_s1_b),
+					 vaddq_u32(v_s1_c, v_s1_d));
+			v_s2 = vaddq_u32(vaddq_u32(v_s2_a, v_s2_b),
+					 vaddq_u32(v_s2_c, v_s2_d));
+			v_s1_sums = vaddq_u32(vaddq_u32(v_s1_sums_a,
+							v_s1_sums_b),
+					      vaddq_u32(v_s1_sums_c,
+							v_s1_sums_d));
+			v_s2 = vaddq_u32(v_s2, vqshlq_n_u32(v_s1_sums, 6));
+
+			s1 += vaddvq_u32(v_s1);
+			s2 += vaddvq_u32(v_s2);
+		}
+		/*
+		 * Process the last 0 <= n < 64 bytes of the chunk using
+		 * scalar instructions and reduce s1 and s2 mod DIVISOR.
+		 */
+		ADLER32_CHUNK(s1, s2, p, n);
+	}
+	return (s2 << 16) | s1;
+}
+#undef ATTRIBUTES
+#endif /* NEON+dotprod implementation */
+
+#if defined(adler32_arm_neon_dotprod) && defined(__ARM_FEATURE_DOTPROD)
+#define DEFAULT_IMPL	adler32_arm_neon_dotprod
+#else
+static inline adler32_func_t
+arch_select_adler32_func(void)
+{
+	const u32 features MAYBE_UNUSED = get_arm_cpu_features();
+
+#ifdef adler32_arm_neon_dotprod
+	if (HAVE_NEON(features) && HAVE_DOTPROD(features))
+		return adler32_arm_neon_dotprod;
+#endif
+#ifdef adler32_arm_neon
+	if (HAVE_NEON(features))
+		return adler32_arm_neon;
+#endif
+	return NULL;
+}
+#define arch_select_adler32_func	arch_select_adler32_func
+#endif
+
+#endif /* LIB_ARM_ADLER32_IMPL_H */